From 2647a1b1ac035b527a77fc12947c020e136ab7d6 Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 3 Jan 2024 16:05:26 +0800 Subject: [PATCH 01/81] =?UTF-8?q?=E5=A2=9E=E5=8A=A0TS-4411bug=E7=9A=84case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/army/community/query/fill/fill_desc.py | 58 ++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/army/community/query/fill/fill_desc.py diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py new file mode 100644 index 0000000000..10b93079e4 --- /dev/null +++ b/tests/army/community/query/fill/fill_desc.py @@ -0,0 +1,58 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + dbname = "db" + stbname = "ocloud_point" + tbname = "ocloud_point_170658_3837620225_1701134595725266945" + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + tdSql.execute( + f'''create stable if not exists {dbname}.{stbname} + (wstart timestamp, point_value float) tags (location binary(64), groupId int) + ''' + ) + + tdSql.execute( + f'''create table if not exists {dbname}.{tbname} using {dbname}.{stbname} tags("California.SanFrancisco", 2)''' + ) + + tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:35:00.000', 5.0)") + tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:36:00.000', 5.0)") + tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:37:00.000', 5.0)") + tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:38:00.000', null)") + tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:39:00.000', 5.0)") + tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:40:00.000', null)") + + tdLog.printNoPrefix("==========step3:fill data") + + tdSql.query(f"wstart as ts, first(point_value) as pointValu from {dbname}.{tbname} where ts between '2823-12-2510:35:00' and '2023-12-2510:40:00' fill(prev) order by wstart desc limit 100") + tdSql.checkRows(6) + tdSql.checkData(0, 1, 5) + tdSql.checkData(1, 1, 5) + tdSql.checkData(2, 1, 5) + tdSql.checkData(3, 1, 5) + tdSql.checkData(4, 1, 5) + tdSql.checkData(5, 1, 5) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 889bea29b406dd776e513022f30415bf210c7c19 Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 3 Jan 2024 16:20:34 +0800 Subject: [PATCH 02/81] =?UTF-8?q?=E5=A2=9E=E5=8A=A0TS-4411bug=E7=9A=84case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/army/community/query/fill/fill_desc.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py index 10b93079e4..7ef8155b1e 100644 --- a/tests/army/community/query/fill/fill_desc.py +++ b/tests/army/community/query/fill/fill_desc.py @@ -1,11 +1,13 @@ import taos import sys -from util.log import * -from util.sql import * -from util.cases import * +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * -class TDTestCase: +class TDTestCase(TBase): def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) @@ -41,7 +43,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step3:fill data") - tdSql.query(f"wstart as ts, first(point_value) as pointValu from {dbname}.{tbname} where ts between '2823-12-2510:35:00' and '2023-12-2510:40:00' fill(prev) order by wstart desc limit 100") + tdSql.query(f"wstart as ts, first(point_value) as pointValu from {dbname}.{tbname} where wstart between '2823-12-2510:35:00' and '2023-12-2510:40:00' fill(prev) order by wstart desc limit 100") tdSql.checkRows(6) tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 5) From 78185b84b947302d1bb72eaaecbabc77677dac15 Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 3 Jan 2024 16:25:31 +0800 Subject: [PATCH 03/81] =?UTF-8?q?=E5=A2=9E=E5=8A=A0TS-4411bug=E7=9A=84case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/army/community/query/fill/fill_desc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py index 7ef8155b1e..efb13692d0 100644 --- a/tests/army/community/query/fill/fill_desc.py +++ b/tests/army/community/query/fill/fill_desc.py @@ -43,7 +43,7 @@ class TDTestCase(TBase): tdLog.printNoPrefix("==========step3:fill data") - tdSql.query(f"wstart as ts, first(point_value) as pointValu from {dbname}.{tbname} where wstart between '2823-12-2510:35:00' and '2023-12-2510:40:00' fill(prev) order by wstart desc limit 100") + tdSql.query(f"select wstart as ts, first(point_value) as pointValue from {dbname}.{tbname} where wstart between '2823-12-2510:35:00' and '2023-12-2510:40:00' fill(prev) order by wstart desc limit 100") tdSql.checkRows(6) tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 5) From f4aedccb80b7bc489b7a4995260984f4a1e88dc4 Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 3 Jan 2024 16:38:55 +0800 Subject: [PATCH 04/81] =?UTF-8?q?=E5=A2=9E=E5=8A=A0TS-4411bug=E7=9A=84case?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/army/community/query/fill/fill_desc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py index efb13692d0..96ae8a4344 100644 --- a/tests/army/community/query/fill/fill_desc.py +++ b/tests/army/community/query/fill/fill_desc.py @@ -43,7 +43,7 @@ class TDTestCase(TBase): tdLog.printNoPrefix("==========step3:fill data") - tdSql.query(f"select wstart as ts, first(point_value) as pointValue from {dbname}.{tbname} where wstart between '2823-12-2510:35:00' and '2023-12-2510:40:00' fill(prev) order by wstart desc limit 100") + tdSql.query(f"select wstart as ts, first(point_value) as pointValue from {dbname}.{tbname} where wstart between '2023-12-26 10:35:00' and '2023-12-26 10:40:00' interval(1M) fill(prev) order by wstart desc limit 100") tdSql.checkRows(6) tdSql.checkData(0, 1, 5) tdSql.checkData(1, 1, 5) From 25e2f30d0e08c303b9980fbc7ae117a944252fb6 Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 3 Jan 2024 17:19:17 +0800 Subject: [PATCH 05/81] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=B9=B6=E8=A1=8C?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/parallel_test/cases.task | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 093fec78ab..a44fcacdb4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,7 +11,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 - +,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py # # system test From 526d51d27f4c7ee5e4fa1e0e19d4099980454c05 Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 9 Jan 2024 10:11:19 +0800 Subject: [PATCH 06/81] use checkDataMem --- tests/army/community/query/fill/fill_desc.py | 12 +++--- tests/army/frame/sql.py | 44 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py index 96ae8a4344..3b11e2b768 100644 --- a/tests/army/community/query/fill/fill_desc.py +++ b/tests/army/community/query/fill/fill_desc.py @@ -44,13 +44,11 @@ class TDTestCase(TBase): tdLog.printNoPrefix("==========step3:fill data") tdSql.query(f"select wstart as ts, first(point_value) as pointValue from {dbname}.{tbname} where wstart between '2023-12-26 10:35:00' and '2023-12-26 10:40:00' interval(1M) fill(prev) order by wstart desc limit 100") - tdSql.checkRows(6) - tdSql.checkData(0, 1, 5) - tdSql.checkData(1, 1, 5) - tdSql.checkData(2, 1, 5) - tdSql.checkData(3, 1, 5) - tdSql.checkData(4, 1, 5) - tdSql.checkData(5, 1, 5) + data = [] + for i in : range(6) # 将csv 文件中的数据保存到data中 + row = [i, 1, 5] + data.append(row) # 选择某一列加入到data数组中 + tdSql.checkDataMem(data) def stop(self): tdSql.close() diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 2e14f0c2f0..19839e217e 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -374,6 +374,50 @@ class TDSql: if(show): tdLog.info("check successfully") + def checkDataMem(self, mem): + if not isinstance(mem, list): + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql) + tdLog.exit("%s(%d) failed: sql:%s, expect data is error, must is array[][]" % args) + + if len(mem) != self.queryRows: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, len(mem), self.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + # row, col, data + for row, rowData in enumerate(mem): + for col, colData in enumerate(rowData): + self.checkData(row, col, colData) + tdLog.info("check successfully") + + def checkDataCsv(self, csvfilePath): + if not isinstance(csvfilePath, str) or len(csvfilePath) == 0: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, csvfilePath) + tdLog.exit("%s(%d) failed: sql:%s, expect csvfile path error:%s" % args) + + tdLog.info("read csvfile read begin") + data = [] + try: + with open(csvfilePath) as csvfile: + csv_reader = csv.reader(csvfile) # 使用csv.reader读取csvfile中的文件 + # header = next(csv_reader) # 读取第一行每一列的标题 + for row in csv_reader: # 将csv 文件中的数据保存到data中 + data.append(row) # 选择某一列加入到data数组中 + except FileNotFoundError: + # 当文件不存在时会引发FileNotFoundError异常 + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, csvfilePath) + tdLog.exit("%s(%d) failed: sql:%s, expect csvfile not find error:%s" % args) + except Exception as e: + # 其他任意类型的异常都将被捕获并输出相应信息 + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, csvfilePath, str(e)) + tdLog.exit("%s(%d) failed: sql:%s, expect csvfile path:%s, read error:%s" % args) + + tdLog.info("read csvfile read successfully") + self.checkDataMem(data) + # return true or false replace exit, no print out def checkRowColNoExit(self, row, col): caller = inspect.getframeinfo(inspect.stack()[2][0]) From 2d3fdf500ea993d0417dff9880a054145c34faf8 Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 9 Jan 2024 14:42:25 +0800 Subject: [PATCH 07/81] use checkDataMem --- tests/army/community/query/fill/fill_desc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py index 3b11e2b768..e30c197f9e 100644 --- a/tests/army/community/query/fill/fill_desc.py +++ b/tests/army/community/query/fill/fill_desc.py @@ -45,9 +45,9 @@ class TDTestCase(TBase): tdSql.query(f"select wstart as ts, first(point_value) as pointValue from {dbname}.{tbname} where wstart between '2023-12-26 10:35:00' and '2023-12-26 10:40:00' interval(1M) fill(prev) order by wstart desc limit 100") data = [] - for i in : range(6) # 将csv 文件中的数据保存到data中 + for i in range(6): row = [i, 1, 5] - data.append(row) # 选择某一列加入到data数组中 + data.append(row) tdSql.checkDataMem(data) def stop(self): From a4bb1adba88d28e25b74ae12245dfd3dfa8d3fe7 Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 10 Jan 2024 09:23:24 +0800 Subject: [PATCH 08/81] modify param --- tests/army/frame/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 19839e217e..b73877ab81 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -387,7 +387,7 @@ class TDSql: # row, col, data for row, rowData in enumerate(mem): for col, colData in enumerate(rowData): - self.checkData(row, col, colData) + self.checkData(row, colData[1], colData[2]) tdLog.info("check successfully") def checkDataCsv(self, csvfilePath): From 5dff14bad8edb3a8da7a77ba83c27e368863286b Mon Sep 17 00:00:00 2001 From: menshibin Date: Wed, 10 Jan 2024 10:03:44 +0800 Subject: [PATCH 09/81] modify param --- tests/army/community/query/fill/fill_desc.py | 4 ++-- tests/army/frame/sql.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py index e30c197f9e..61638e73ed 100644 --- a/tests/army/community/query/fill/fill_desc.py +++ b/tests/army/community/query/fill/fill_desc.py @@ -43,10 +43,10 @@ class TDTestCase(TBase): tdLog.printNoPrefix("==========step3:fill data") - tdSql.query(f"select wstart as ts, first(point_value) as pointValue from {dbname}.{tbname} where wstart between '2023-12-26 10:35:00' and '2023-12-26 10:40:00' interval(1M) fill(prev) order by wstart desc limit 100") + tdSql.query(f"select first(point_value) as pointValue from {dbname}.{tbname} where wstart between '2023-12-26 10:35:00' and '2023-12-26 10:40:00' interval(1M) fill(prev) order by wstart desc limit 100") data = [] for i in range(6): - row = [i, 1, 5] + row = [5] data.append(row) tdSql.checkDataMem(data) diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index b73877ab81..19839e217e 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -387,7 +387,7 @@ class TDSql: # row, col, data for row, rowData in enumerate(mem): for col, colData in enumerate(rowData): - self.checkData(row, colData[1], colData[2]) + self.checkData(row, col, colData) tdLog.info("check successfully") def checkDataCsv(self, csvfilePath): From 006a3c38045ce840fd8eec6d396d46d0cab710d4 Mon Sep 17 00:00:00 2001 From: menshibin Date: Fri, 12 Jan 2024 14:14:03 +0800 Subject: [PATCH 10/81] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 104 ++++++++++++++++++++ tests/army/frame/autogen.py | 61 ++++++------ tests/army/frame/caseBase.py | 14 +-- tests/army/frame/server/cluster.py | 11 ++- tests/army/frame/server/dnodes.py | 5 + tests/army/test.py | 21 ++-- 6 files changed, 165 insertions(+), 51 deletions(-) create mode 100644 tests/army/community/cluster/incSnapshot.py diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py new file mode 100644 index 0000000000..a91e51cda4 --- /dev/null +++ b/tests/army/community/cluster/incSnapshot.py @@ -0,0 +1,104 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame.server.dnodes import * +from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=3): + super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1") + self.valgrind = 0 + self.childtable_count = 10 + # tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.prepare() + autoGen = AutoGen() + autoGen.create_db(self.db, 2, 3) + autoGen.create_stable(self.stb, 5, 10, 8, 8) + autoGen.create_child(self.stb, "d", self.childtable_count) + autoGen.insert_data(1) + tdSql.execute(f"flush database {self.db}") + clusterDnodes.stoptaosd(3) + clusterDnodes.stoptaosd(1) + clusterDnodes.starttaosd(3) + time.sleep(5) + clusterDnodes.stoptaosd(2) + clusterDnodes.starttaosd(1) + time.sleep(5) + autoGen.insert_data(10, 1600000000001) + # tdSql.execute(f"flush database {self.db}") + + sql = 'show vnodes;' + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + self.snapshotAgg() + + clusterDnodes.stopAll() + # for i in range(1, 4): + # path = clusterDnodes.getDnodeDir(i) + # dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") + # dirs = glob.glob(dnodesRootDir) + # for dir in dirs: + # if os.path.isdir(dir): + # tdLog.debug("delete dir: %s " % (dnodesRootDir)) + # self.remove_directory(os.path.join(dir, "wal")) + + clusterDnodes.starttaosd(1) + clusterDnodes.starttaosd(2) + clusterDnodes.starttaosd(3) + + time.sleep(3) + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'offline': + tdLog.exit( + "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1])) + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + + self.timestamp_step = 1 + self.insert_rows = 11 + self.checkInsertCorrect() + self.checkAggCorrect() + + def remove_directory(self, directory): + try: + shutil.rmtree(directory) + tdLog.debug("delete dir: %s " % (directory)) + except OSError as e: + tdLog.exit("delete fail dir: %s " % (directory)) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index 9dca96e7b0..a641deaf00 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -30,7 +30,7 @@ class AutoGen: # _columns_sql def gen_columns_sql(self, pre, cnt, binary_len, nchar_len): - types = [ + types = [ 'timestamp', 'tinyint', 'smallint', @@ -58,33 +58,33 @@ class AutoGen: sqls += "," sqls += sql metas.append(sel) - - return metas, sqls; + + return metas, sqls; # gen tags data def gen_data(self, i, marr): - datas = "" + datas = "" for c in marr: data = "" - if c == 0 : # timestamp + if c == 0: # timestamp data = "%d" % (self.ts + i) - elif c <= 4 : # small - data = "%d"%(i%128) - elif c <= 8 : # int + elif c <= 4: # small + data = "%d" % (i % 128) + elif c <= 8: # int data = f"{i}" - elif c <= 10 : # float - data = "%f"%(i+i/1000) - elif c <= 11 : # bool - data = "%d"%(i%2) - elif c == 12 : # binary + elif c <= 10: # float + data = "%f" % (i + i / 1000) + elif c <= 11: # bool + data = "%d" % (i % 2) + elif c == 12: # binary data = '"' + self.random_string(self.bin_len) + '"' - elif c == 13 : # binary + elif c == 13: # binary data = '"' + self.random_string(self.nch_len) + '"' if datas != "": datas += "," datas += data - + return datas # generate specail wide random string @@ -93,11 +93,11 @@ class AutoGen: return ''.join(random.choice(letters) for i in range(count)) # create db - def create_db(self, dbname, vgroups = 2, replica = 1): - self.dbname = dbname + def create_db(self, dbname, vgroups=2, replica=1): + self.dbname = dbname tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}') tdSql.execute(f'use {dbname}') - + # create table or stable def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len): self.bin_len = binary_len @@ -109,7 +109,7 @@ class AutoGen: sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})" tdSql.execute(sql) - # create child table + # create child table def create_child(self, stbname, prename, cnt): self.child_cnt = cnt self.child_name = prename @@ -120,7 +120,7 @@ class AutoGen: tdLog.info(f"create child tables {cnt} ok") - def insert_data_child(self, child_name, cnt, batch_size, step): + def insert_data_child(self, child_name, cnt, batch_size, step): values = "" print("insert child data") ts = self.ts @@ -130,7 +130,7 @@ class AutoGen: value = self.gen_data(i, self.mcols) ts += step values += f"({ts},{value}) " - if batch_size == 1 or (i > 0 and i % batch_size == 0) : + if batch_size == 1 or (i > 0 and i % batch_size == 0): sql = f"insert into {child_name} values {values}" tdSql.execute(sql) values = "" @@ -138,18 +138,25 @@ class AutoGen: # end batch if values != "": sql = f"insert into {child_name} values {values}" + tdLog.info(f" insert child data SQL{sql}") tdSql.execute(sql) tdLog.info(f" insert data i={i}") values = "" - tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + tdLog.info(f" insert child data {child_name} finished, insert rows={cnt} last_ts={ts}") + return ts - # insert data - def insert_data(self, cnt): + # insert data + def insert_data(self, cnt, bContinue=False): + if not bContinue: + self.ts = 1600000000000 + + currTs = 1600000000000 for i in range(self.child_cnt): name = f"{self.child_name}{i}" - self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") # insert same timestamp to all childs @@ -158,6 +165,4 @@ class AutoGen: name = f"{self.child_name}{i}" self.insert_data_child(name, cnt, self.batch_size, 0) - tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}") - - + tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 8d2c1e6d18..863a117cdd 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -28,7 +28,7 @@ class TBase: # # init - def init(self, conn, logSql, replicaVar=1): + def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"): # save param self.replicaVar = int(replicaVar) tdSql.init(conn.cursor(), True) @@ -40,14 +40,14 @@ class TBase: self.mLevelDisk = 0 # test case information - self.db = "db" - self.stb = "stb" + self.db = db + self.stb = stb # sql - self.sqlSum = f"select sum(ic) from {self.stb}" - self.sqlMax = f"select max(ic) from {self.stb}" - self.sqlMin = f"select min(ic) from {self.stb}" - self.sqlAvg = f"select avg(ic) from {self.stb}" + self.sqlSum = f"select sum({checkColName}) from {self.stb}" + self.sqlMax = f"select max({checkColName}) from {self.stb}" + self.sqlMin = f"select min({checkColName}) from {self.stb}" + self.sqlAvg = f"select avg({checkColName}) from {self.stb}" self.sqlFirst = f"select first(ts) from {self.stb}" self.sqlLast = f"select last(ts) from {self.stb}" diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py index ade8ac39a2..9be7fe56b1 100644 --- a/tests/army/frame/server/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -13,23 +13,24 @@ from frame.common import * class ClusterDnodes(TDDnodes): """rewrite TDDnodes and make MyDdnodes as TDDnodes child class""" - def __init__(self ,dnodes_lists): - + def __init__(self): super(ClusterDnodes,self).__init__() - self.dnodes = dnodes_lists # dnode must be TDDnode instance self.simDeployed = False self.testCluster = False self.valgrind = 0 self.killValgrind = 1 + def init(self, dnodes_lists, deployPath, masterIp): + self.dnodes = dnodes_lists # dnode must be TDDnode instance + super(ClusterDnodes, self).init(deployPath, masterIp) +clusterDnodes = ClusterDnodes() class ConfigureyCluster: """This will create defined number of dnodes and create a cluster. at the same time, it will return TDDnodes list: dnodes, """ hostname = socket.gethostname() - def __init__(self): - self.dnodes = [] + self.dnodes = [] self.dnodeNums = 5 self.independent = True self.startPort = 6030 diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 0d40b665dd..b58deeb789 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -899,6 +899,11 @@ class TDDnodes: dnodesRootDir = "%s/sim" % (self.path) return dnodesRootDir + def getDnodeDir(self, index): + self.check(index) + dnodesDir = "%s/sim/dnode%d" % (self.path, index) + return dnodesDir + def getSimCfgPath(self): return self.sim.getCfgDir() diff --git a/tests/army/test.py b/tests/army/test.py index a3e28b772d..37fecea370 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -405,11 +405,11 @@ if __name__ == "__main__": else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() for dnode in tdDnodes.dnodes: tdDnodes.deploy(dnode.index, updateCfgDict) for dnode in tdDnodes.dnodes: @@ -591,12 +591,11 @@ if __name__ == "__main__": print(independentMnode,"independentMnode valuse") # create dnode list dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.setAsan(asan) - tdDnodes.stopAll() + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() for dnode in tdDnodes.dnodes: tdDnodes.deploy(dnode.index,updateCfgDict) for dnode in tdDnodes.dnodes: From 2be6098a2e3e7fdd539ccb6e9e029f14f0fd00cb Mon Sep 17 00:00:00 2001 From: menshibin Date: Mon, 15 Jan 2024 14:59:09 +0800 Subject: [PATCH 11/81] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 58 ++++++++++----------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index a91e51cda4..94013eea70 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -31,45 +31,45 @@ class TDTestCase(TBase): autoGen.create_db(self.db, 2, 3) autoGen.create_stable(self.stb, 5, 10, 8, 8) autoGen.create_child(self.stb, "d", self.childtable_count) - autoGen.insert_data(1) + autoGen.insert_data(1000) tdSql.execute(f"flush database {self.db}") clusterDnodes.stoptaosd(3) - clusterDnodes.stoptaosd(1) - clusterDnodes.starttaosd(3) - time.sleep(5) - clusterDnodes.stoptaosd(2) - clusterDnodes.starttaosd(1) - time.sleep(5) - autoGen.insert_data(10, 1600000000001) - # tdSql.execute(f"flush database {self.db}") + # clusterDnodes.stoptaosd(1) + # clusterDnodes.starttaosd(3) + # time.sleep(5) + # clusterDnodes.stoptaosd(2) + # clusterDnodes.starttaosd(1) + # time.sleep(5) + autoGen.insert_data(5000, 1600000000001) + tdSql.execute(f"flush database {self.db}") - sql = 'show vnodes;' - while True: - bFinish = True - param_list = tdSql.query(sql, row_tag=True) - for param in param_list: - if param[3] == 'leading' or param[3] == 'following': - bFinish = False - break - if bFinish: - break + # sql = 'show vnodes;' + # while True: + # bFinish = True + # param_list = tdSql.query(sql, row_tag=True) + # for param in param_list: + # if param[3] == 'leading' or param[3] == 'following': + # bFinish = False + # break + # if bFinish: + # break self.snapshotAgg() - + time.sleep(10) clusterDnodes.stopAll() - # for i in range(1, 4): - # path = clusterDnodes.getDnodeDir(i) - # dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") - # dirs = glob.glob(dnodesRootDir) - # for dir in dirs: - # if os.path.isdir(dir): - # tdLog.debug("delete dir: %s " % (dnodesRootDir)) - # self.remove_directory(os.path.join(dir, "wal")) + for i in range(1, 4): + path = clusterDnodes.getDnodeDir(i) + dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") + dirs = glob.glob(dnodesRootDir) + for dir in dirs: + if os.path.isdir(dir): + tdLog.debug("delete dir: %s " % (dnodesRootDir)) + self.remove_directory(os.path.join(dir, "wal")) clusterDnodes.starttaosd(1) clusterDnodes.starttaosd(2) clusterDnodes.starttaosd(3) - time.sleep(3) + time.sleep(10) while True: bFinish = True param_list = tdSql.query(sql, row_tag=True) From 764365047d07bc83ffff50c10bac1ad179665fd1 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 16 Jan 2024 09:28:45 +0800 Subject: [PATCH 12/81] fix: stream scan core due to table end index introduced in 1 null row for empty group --- source/libs/executor/inc/executorInt.h | 2 ++ source/libs/executor/src/executor.c | 10 ++++++---- source/libs/executor/src/scanoperator.c | 19 ++++++++++++------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index c3f47cde9d..2ea23e73f2 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -273,6 +273,8 @@ typedef struct STableScanInfo { int32_t tableStartIndex; // current group scan start int32_t tableEndIndex; // current group scan end int32_t currentGroupIndex; // current group index of groupOffset + int32_t currentGroupId; + int32_t currentTable; int8_t scanMode; int8_t assignBlockUid; uint8_t countState; // empty table count state diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index eb84cb0639..1bccc514e4 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1264,6 +1264,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pTableInfo = tableListGetInfo(pTableListInfo, 0); uid = pTableInfo->uid; ts = INT64_MIN; + pScanInfo->currentTable = 0; pScanInfo->tableEndIndex = 0; } else { taosRUnLockLatch(&pTaskInfo->lock); @@ -1278,16 +1279,17 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pInfo->pTableScanOp->resultInfo.totalRows = 0; // start from current accessed position - // we cannot start from the pScanInfo->tableEndIndex, since the commit offset may cause the rollback of the start + // we cannot start from the pScanInfo->currentTable, since the commit offset may cause the rollback of the start // position, let's find it from the beginning. index = tableListFind(pTableListInfo, uid, 0); taosRUnLockLatch(&pTaskInfo->lock); if (index >= 0) { + pScanInfo->currentTable = index; pScanInfo->tableEndIndex = index; } else { qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, - numOfTables, pScanInfo->tableEndIndex, id); + numOfTables, pScanInfo->currentTable, id); terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1310,12 +1312,12 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } qDebug("tsdb reader created with offset(snapshot) uid:%" PRId64 " ts:%" PRId64 " table index:%d, total:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } else { pTaskInfo->storageAPI.tsdReader.tsdSetQueryTableList(pScanBaseInfo->dataReader, &keyInfo, 1); pTaskInfo->storageAPI.tsdReader.tsdReaderResetStatus(pScanBaseInfo->dataReader, &pScanBaseInfo->cond); qDebug("tsdb reader offset seek snapshot to uid:%" PRId64 " ts %" PRId64 " table index:%d numOfTable:%d, %s", - uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->tableEndIndex, numOfTables, id); + uid, pScanBaseInfo->cond.twindows.skey, pScanInfo->currentTable, numOfTables, id); } // restore the key value diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3ed5128858..6736d6e137 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -939,7 +939,7 @@ static SSDataBlock* startNextGroupScan(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStorageAPI* pAPI = &pTaskInfo->storageAPI; int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if (pInfo->tableEndIndex + 1 >= numOfTables) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { setOperatorCompleted(pOperator); if (pOperator->dynamicTask) { taosArrayClear(pInfo->base.pTableListInfo->pTableList); @@ -978,9 +978,9 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { int32_t num = 0; STableKeyInfo* pList = NULL; - if (pInfo->tableEndIndex == -1) { + if (pInfo->currentGroupId == -1) { int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if (pInfo->tableEndIndex + 1 == numOfTables) { + if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) { setOperatorCompleted(pOperator); return NULL; } @@ -1034,7 +1034,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, code); } if (pOperator->status == OP_EXEC_DONE) { - pInfo->tableEndIndex = -1; + pInfo->currentGroupId = -1; pOperator->status = OP_OPENED; SSDataBlock* result = NULL; while (true) { @@ -1059,23 +1059,24 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { } // if no data, switch to next table and continue scan + pInfo->currentTable++; pInfo->tableEndIndex++; taosRLockLatch(&pTaskInfo->lock); numOfTables = tableListGetSize(pInfo->base.pTableListInfo); - if (pInfo->tableEndIndex >= numOfTables) { + if (pInfo->currentTable >= numOfTables) { qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo)); taosRUnLockLatch(&pTaskInfo->lock); return NULL; } - tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableEndIndex); + tInfo = *(STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable); taosRUnLockLatch(&pTaskInfo->lock); pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1); qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables, - pInfo->tableEndIndex, numOfTables, GET_TASKID(pTaskInfo)); + pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo)); pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond); pInfo->scanTimes = 0; @@ -1167,6 +1168,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, if (code != TSDB_CODE_SUCCESS) { goto _error; } + + pInfo->currentGroupId = -1; pInfo->tableEndIndex = -1; pInfo->currentGroupIndex = -1; @@ -1264,6 +1267,7 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, uint6 pTableScanInfo->base.cond.startVersion = 0; pTableScanInfo->base.cond.endVersion = ver; pTableScanInfo->scanTimes = 0; + pTableScanInfo->currentGroupId = -1; pTableScanInfo->tableEndIndex = -1; pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader); pTableScanInfo->base.dataReader = NULL; @@ -2167,6 +2171,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pTableScanOp->status = OP_OPENED; pTSInfo->scanTimes = 0; + pTSInfo->currentGroupId = -1; pTSInfo->tableEndIndex = -1; } From 1119d95a4aa9166f9f06bc4a4a3571c5c7cf4c71 Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 14:38:53 +0800 Subject: [PATCH 13/81] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 2 +- tests/parallel_test/cases.task | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 94013eea70..21f67b45fc 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -40,7 +40,7 @@ class TDTestCase(TBase): # clusterDnodes.stoptaosd(2) # clusterDnodes.starttaosd(1) # time.sleep(5) - autoGen.insert_data(5000, 1600000000001) + autoGen.insert_data(5000, True) tdSql.execute(f"flush database {self.db}") # sql = 'show vnodes;' diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 63745f75ab..5e8921768d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,6 +11,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py From 9a7ef7fa182f3a350060bd55ba63743ae9e7bf15 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 16 Jan 2024 15:08:50 +0800 Subject: [PATCH 14/81] fix: reuse tableListGetGroupList --- source/libs/executor/src/executil.c | 4 --- source/libs/executor/src/scanoperator.c | 39 ++++++++++++------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1da7818c3c..19f33d2420 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -646,10 +646,6 @@ int32_t getColInfoResultForGroupby(void* pVnode, SNodeList* group, STableListInf } } - if (initRemainGroups) { - pTableListInfo->numOfOuputGroups = taosHashGetSize(pTableListInfo->remainGroups); - } - if (tsTagFilterCache) { tableList = taosArrayDup(pTableListInfo->pTableList, NULL); pAPI->metaFn.metaPutTbGroupToCache(pVnode, pTableListInfo->idInfo.suid, context.digest, tListLen(context.digest), diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 6736d6e137..36a7f4c183 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -657,33 +657,17 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) { - pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, pKeyInfo, size); - STableListInfo* pTableListInfo = pInfo->base.pTableListInfo; - int32_t numOfTables = tableListGetSize(pTableListInfo); - STableKeyInfo* pStart = (STableKeyInfo*)tableListGetInfo(pTableListInfo, pInfo->tableStartIndex); + pInfo->tableStartIndex = TARRAY_ELEM_IDX(pInfo->base.pTableListInfo->pTableList, *pKeyInfo); - if (pTableListInfo->oneTableForEachGroup) { - pInfo->tableEndIndex = pInfo->tableStartIndex; - } else if (pTableListInfo->groupOffset) { - pInfo->currentGroupIndex++; - if (pInfo->currentGroupIndex + 1 < pTableListInfo->numOfOuputGroups) { - pInfo->tableEndIndex = pTableListInfo->groupOffset[pInfo->currentGroupIndex + 1] - 1; - } else { - pInfo->tableEndIndex = numOfTables - 1; - } - } else { - pInfo->tableEndIndex = numOfTables - 1; - } + pInfo->tableEndIndex = (pInfo->tableStartIndex + (*size) - 1); if (!pInfo->needCountEmptyTable) { pInfo->countState = TABLE_COUNT_STATE_END; } else { pInfo->countState = TABLE_COUNT_STATE_SCAN; } - - *pKeyInfo = pStart; - *size = pInfo->tableEndIndex - pInfo->tableStartIndex + 1; } void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { @@ -984,7 +968,8 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { setOperatorCompleted(pOperator); return NULL; } - + + initNextGroupScan(pInfo, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); @@ -1006,16 +991,28 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; } + uInfo("slzhou 1 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows); return result; } while (true) { result = startNextGroupScan(pOperator); if (result || pOperator->status == OP_EXEC_DONE) { + if (result) { + uInfo("slzhou 2 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows); + } + else { + uInfo("slzhou 2 null block" ); + } return result; } } - + if (result) { + uInfo("slzhou 3 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows); + } + else { + uInfo("slzhou 3 null block" ); + } return result; } From de6b559ab9bd1302bd4fb610868abd0cff524edc Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 16 Jan 2024 15:21:20 +0800 Subject: [PATCH 15/81] fix: remove some uage of table end index --- source/libs/executor/inc/executorInt.h | 1 - source/libs/executor/src/executor.c | 2 -- source/libs/executor/src/scanoperator.c | 3 --- 3 files changed, 6 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 2ea23e73f2..007eab40f8 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -272,7 +272,6 @@ typedef struct STableScanInfo { SSampleExecInfo sample; // sample execution info int32_t tableStartIndex; // current group scan start int32_t tableEndIndex; // current group scan end - int32_t currentGroupIndex; // current group index of groupOffset int32_t currentGroupId; int32_t currentTable; int8_t scanMode; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1bccc514e4..e872c87237 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1265,7 +1265,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT uid = pTableInfo->uid; ts = INT64_MIN; pScanInfo->currentTable = 0; - pScanInfo->tableEndIndex = 0; } else { taosRUnLockLatch(&pTaskInfo->lock); qError("no table in table list, %s", id); @@ -1286,7 +1285,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (index >= 0) { pScanInfo->currentTable = index; - pScanInfo->tableEndIndex = index; } else { qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, numOfTables, pScanInfo->currentTable, id); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 36a7f4c183..103166e3e7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1057,7 +1057,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // if no data, switch to next table and continue scan pInfo->currentTable++; - pInfo->tableEndIndex++; taosRLockLatch(&pTaskInfo->lock); numOfTables = tableListGetSize(pInfo->base.pTableListInfo); @@ -1169,7 +1168,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->currentGroupId = -1; pInfo->tableEndIndex = -1; - pInfo->currentGroupIndex = -1; pInfo->assignBlockUid = pTableScanNode->assignBlockUid; pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false; @@ -2169,7 +2167,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; - pTSInfo->tableEndIndex = -1; } if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) { From 7db577d664d4ec7d69678210a1f3f178eab511de Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 15:28:19 +0800 Subject: [PATCH 16/81] add inc Snapshot copy case --- tests/army/test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/army/test.py b/tests/army/test.py index 37fecea370..62b3b46a61 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -410,10 +410,10 @@ if __name__ == "__main__": clusterDnodes.setValgrind(valgrind) clusterDnodes.setAsan(asan) clusterDnodes.stopAll() - for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index, updateCfgDict) - for dnode in tdDnodes.dnodes: - tdDnodes.starttaosd(dnode.index) + for dnode in clusterDnodes.dnodes: + clusterDnodes.deploy(dnode.index, updateCfgDict) + for dnode in clusterDnodes.dnodes: + clusterDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) if restful or websocket: @@ -596,10 +596,10 @@ if __name__ == "__main__": clusterDnodes.setValgrind(valgrind) clusterDnodes.setAsan(asan) clusterDnodes.stopAll() - for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index,updateCfgDict) - for dnode in tdDnodes.dnodes: - tdDnodes.starttaosd(dnode.index) + for dnode in clusterDnodes.dnodes: + clusterDnodes.deploy(dnode.index,updateCfgDict) + for dnode in clusterDnodes.dnodes: + clusterDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) if restful or websocket: From 24066a7734cabcf4070e61633cba92bed78654cc Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 15:35:33 +0800 Subject: [PATCH 17/81] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 21f67b45fc..2095b6008b 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -68,7 +68,7 @@ class TDTestCase(TBase): clusterDnodes.starttaosd(1) clusterDnodes.starttaosd(2) clusterDnodes.starttaosd(3) - + sql = "show vnodes;" time.sleep(10) while True: bFinish = True From 022965661a7fcd0fa0a04823105b72aef7daeaad Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 15:38:34 +0800 Subject: [PATCH 18/81] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 2095b6008b..9f50ea52f1 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -84,7 +84,7 @@ class TDTestCase(TBase): break self.timestamp_step = 1 - self.insert_rows = 11 + self.insert_rows = 6000 self.checkInsertCorrect() self.checkAggCorrect() From 8edff6e35593ced01718207dea26d7b4c0688470 Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 15:51:16 +0800 Subject: [PATCH 19/81] add inc Snapshot copy case --- tests/army/frame/autogen.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index 1e041f633d..d38d2e2f19 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -162,16 +162,23 @@ class AutoGen: tdLog.info(f" insert data i={i}") values = "" - tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + return ts - # insert data - def insert_data(self, cnt): + def insert_data(self, cnt, bContinue=False): + if not bContinue: + self.ts = 1600000000000 + + currTs = 1600000000000 for i in range(self.child_cnt): name = f"{self.child_name}{i}" - self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") + # insert same timestamp to all childs + # insert same timestamp to all childs def insert_samets(self, cnt): for i in range(self.child_cnt): From 449c1631ef04224b67193a6069044d35f63f3317 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 17 Jan 2024 13:34:37 +0800 Subject: [PATCH 20/81] fix: add numOfOutputGroups back when group order scan --- source/libs/executor/src/executil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 19f33d2420..fd34a98eca 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2155,6 +2155,8 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* return code; } + if (pScanNode->groupOrderScan) pTableListInfo->numOfOuputGroups = taosArrayGetSize(pTableListInfo->pTableList); + if (groupSort || pScanNode->groupOrderScan) { code = sortTableGroup(pTableListInfo); } From d5d1dd9127e04dc08dbe66666eb718b789fa2a62 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Jan 2024 20:08:26 +0800 Subject: [PATCH 21/81] opti:[TD-28118] raw block data for tmq --- include/common/tmsg.h | 8 ++++++ source/client/inc/clientInt.h | 2 ++ source/client/src/clientImpl.c | 14 +++++++---- source/client/src/clientRawBlockWrite.c | 22 ++++++++++++++--- source/client/src/clientTmq.c | 33 ++++++++++++++++--------- source/common/src/tdatablock.c | 4 +-- source/dnode/vnode/src/tq/tqScan.c | 9 +++---- 7 files changed, 65 insertions(+), 27 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c314d82036..2ee48a18e0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1975,6 +1975,14 @@ typedef struct { char data[]; } SRetrieveTableRsp; +typedef struct { + int64_t version; + int64_t numOfRows; + int8_t compressed; + int8_t precision; + char data[]; +} SRetrieveTableRspForTmq; + typedef struct { int64_t handle; int64_t useconds; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index a6d5039be7..5fb789f0db 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -298,6 +298,8 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool freeAfterUse); +int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, + bool convertUcs4); void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); void doFreeReqResultInfo(SReqResultInfo* pResInfo); int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d4f89d6b54..9800d233e9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1795,6 +1795,7 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; + int32_t blockVersion = *(int32_t*)p; // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | @@ -1810,7 +1811,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i char* pStart = p + len; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = htonl(colLength[i]); + int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; @@ -1873,6 +1874,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int tscDebug("start to convert form json format string"); char* p = (char*)pResultInfo->pData; + int32_t blockVersion = *(int32_t*)p; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); if (dataLen <= 0) { return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -1908,8 +1910,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* pStart = p; char* pStart1 = p1; for (int32_t i = 0; i < numOfCols; ++i) { - int32_t colLen = htonl(colLength[i]); - int32_t colLen1 = htonl(colLength1[i]); + int32_t colLen = (blockVersion == 1) ? htonl(colLength[i]) : colLength[i]; + int32_t colLen1 = (blockVersion == 1) ? htonl(colLength1[i]) : colLength1[i]; if (ASSERT(colLen < dataLen)) { tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; @@ -1968,7 +1970,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int } colLen1 = len; totalLen += colLen1; - colLength1[i] = htonl(len); + colLength1[i] = (blockVersion == 1) ? htonl(len) : len; } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { len = numOfRows * sizeof(int32_t); memcpy(pStart1, pStart, len); @@ -2057,7 +2059,9 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { - colLength[i] = htonl(colLength[i]); + if(blockVersion == 1){ + colLength[i] = htonl(colLength[i]); + } if (colLength[i] >= dataLen) { tscError("invalid colLength %d, dataLen %d", colLength[i], dataLen); return TSDB_CODE_TSC_INTERNAL_ERROR; diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 1f9d3c6d8c..12ee67abbb 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1553,6 +1553,18 @@ end: return code; } +static void* getRawDataFromRes(void *pRetrieve){ + void* rawData = NULL; + // deal with compatibility + if(*(int64_t*)pRetrieve == 0){ + rawData = ((SRetrieveTableRsp*)pRetrieve)->data; + }else if(*(int64_t*)pRetrieve == 1){ + rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; + } + ASSERT(rawData != NULL); + return rawData; +} + static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { if(taos == NULL || data == NULL){ terrno = TSDB_CODE_INVALID_PARA; @@ -1607,7 +1619,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); while (++rspObj.resIter < rspObj.rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); + void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); if (!rspObj.rsp.withSchema) { goto end; } @@ -1653,7 +1665,8 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { fields[i].bytes = pSW->pSchema[i].bytes; tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } - code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, NULL, fields, pSW->nCols, true); + void* rawData = getRawDataFromRes(pRetrieve); + code = rawBlockBindData(pQuery, pTableMeta, rawData, NULL, fields, pSW->nCols, true); taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { goto end; @@ -1737,7 +1750,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) uDebug(LOG_ID_TAG" write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); + void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); if (!rspObj.rsp.withSchema) { goto end; } @@ -1824,7 +1837,8 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) fields[i].bytes = pSW->pSchema[i].bytes; tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } - code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, pCreateReqDst, fields, pSW->nCols, true); + void* rawData = getRawDataFromRes(pRetrieve); + code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, fields, pSW->nCols, true); taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { goto end; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 28e269ee10..d98c3340c1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1577,16 +1577,21 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - if (!pWrapper->dataRsp.withSchema) { - setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); - } - // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i); + SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); int64_t rows = htobe64(pRetrieve->numOfRows); pVg->numOfRows += rows; (*numOfRows) += rows; + + if (!pRspObj->rsp.withSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable + pRspObj->rsp.withSchema = true; + pRspObj->rsp.blockSchema = taosArrayInit(pRspObj->rsp.blockNum, sizeof(void *)); + SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); + if(schema){ + taosArrayPush(pRspObj->rsp.blockSchema, &schema); + } + } } return pRspObj; @@ -1603,13 +1608,10 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClie pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - if (!pWrapper->taosxRsp.withSchema) { - setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); - } // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, i); + SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); int64_t rows = htobe64(pRetrieve->numOfRows); pVg->numOfRows += rows; (*numOfRows) += rows; @@ -2548,7 +2550,7 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { pRspObj->resIter++; if (pRspObj->resIter < pRspObj->rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + SRetrieveTableRspForTmq* pRetrieveTmq = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); if (pRspObj->rsp.withSchema) { SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); @@ -2559,7 +2561,16 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { taosMemoryFreeClear(pRspObj->resInfo.convertJson); } - setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false); + pRspObj->resInfo.pData = (void*)pRetrieveTmq->data; + pRspObj->resInfo.numOfRows = htobe64(pRetrieveTmq->numOfRows); + pRspObj->resInfo.current = 0; + pRspObj->resInfo.precision = pRetrieveTmq->precision; + + // TODO handle the compressed case + pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows; + setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows, + convertUcs4); + return &pRspObj->resInfo; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f0d1630480..0fb066d521 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2196,7 +2196,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { // todo extract method int32_t* version = (int32_t*)data; - *version = 1; + *version = 2; data += sizeof(int32_t); int32_t* actualLen = (int32_t*)data; @@ -2277,7 +2277,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { data += colSizes[col]; } - colSizes[col] = htonl(colSizes[col]); +// colSizes[col] = htonl(colSizes[col]); // uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, // htonl(colSizes[col]), colSizes[col]); } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 01866ef893..5432637482 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -16,21 +16,20 @@ #include "tq.h" int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; - pRetrieve->useconds = 0; + SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf; + pRetrieve->version = 1; pRetrieve->precision = precision; pRetrieve->compressed = 0; - pRetrieve->completed = 1; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); - actualLen += sizeof(SRetrieveTableRsp); + actualLen += sizeof(SRetrieveTableRspForTmq); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); From f1fb7f5c147d112e20f84a07ca4f40d99e25cd7e Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 Jan 2024 08:43:36 +0800 Subject: [PATCH 22/81] fix: change num of output groups when cout empty group is needed --- source/libs/executor/src/executil.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fd34a98eca..033cda43f2 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2138,6 +2138,8 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = numOfTables; } else if (groupByTbname && pScanNode->groupOrderScan){ pTableListInfo->numOfOuputGroups = numOfTables; + } else if (groupByTbname && tsCountAlwaysReturnValue && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) { + pTableListInfo->numOfOuputGroups = numOfTables; } else { pTableListInfo->numOfOuputGroups = 1; } From 4227340d9e69d0a34b3249d93af2324d5166189d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 Jan 2024 09:11:58 +0800 Subject: [PATCH 23/81] opti:[TD-28118] raw block data for tmq --- source/common/src/tdatablock.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0fb066d521..5382259899 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2293,7 +2293,6 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { int32_t version = *(int32_t*)pStart; pStart += sizeof(int32_t); - ASSERT(version == 1); // total length sizeof(int32_t) int32_t dataLen = *(int32_t*)pStart; @@ -2339,7 +2338,9 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { - colLen[i] = htonl(colLen[i]); + if(version == 1){ + colLen[i] = htonl(colLen[i]); + } ASSERT(colLen[i] >= 0); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); From a1f7169b639af0f8852d4d953aa15895903a40ae Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 Jan 2024 09:52:04 +0800 Subject: [PATCH 24/81] fix: remove uInfo --- source/libs/executor/src/scanoperator.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 103166e3e7..09f4f6ac3c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -991,28 +991,16 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) { if (pOperator->dynamicTask) { result->info.id.groupId = result->info.id.uid; } - uInfo("slzhou 1 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows); return result; } while (true) { result = startNextGroupScan(pOperator); if (result || pOperator->status == OP_EXEC_DONE) { - if (result) { - uInfo("slzhou 2 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows); - } - else { - uInfo("slzhou 2 null block" ); - } return result; } } - if (result) { - uInfo("slzhou 3 %lu, %lu, %lu", result->info.id.groupId, result->info.id.uid, result->info.rows); - } - else { - uInfo("slzhou 3 null block" ); - } + return result; } From 4544d06ef411d887e28cce2b1e01f35cc4d232c6 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 18 Jan 2024 10:42:53 +0800 Subject: [PATCH 25/81] add test case for function 'elapsed' by charles --- .../query/function/test_fun_elapsed.py | 418 ++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 419 insertions(+) create mode 100644 tests/army/community/query/function/test_fun_elapsed.py diff --git a/tests/army/community/query/function/test_fun_elapsed.py b/tests/army/community/query/function/test_fun_elapsed.py new file mode 100644 index 0000000000..66775c8e6c --- /dev/null +++ b/tests/army/community/query/function/test_fun_elapsed.py @@ -0,0 +1,418 @@ +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * + + +class TDTestCase(TBase): + """Verify the elapsed function + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.table_dic = { + "super_table": ["st1", "st2", "st_empty"], + "child_table": ["ct1_1", "ct1_2", "ct1_empty", "ct2_1", "ct2_2", "ct2_empty"], + "tags_value": [("2023-03-01 15:00:00", 1, 'bj'), ("2023-03-01 15:10:00", 2, 'sh'), ("2023-03-01 15:20:00", 3, 'sz'), ("2023-03-01 15:00:00", 4, 'gz'), ("2023-03-01 15:10:00", 5, 'cd'), ("2023-03-01 15:20:00", 6, 'hz')], + "common_table": ["t1", "t2", "t_empty"] + } + self.start_ts = 1677654000000 # 2023-03-01 15:00:00.000 + self.row_num = 100 + + def prepareData(self): + # db + tdSql.execute("create database {};".format(self.dbname)) + tdSql.execute("use {};".format(self.dbname)) + tdLog.debug("Create database %s" % self.dbname) + + # commont table + for common_table in self.table_dic["common_table"]: + tdSql.execute("create table {} (ts timestamp, c_ts timestamp, c_int int, c_bigint bigint, c_double double, c_nchar nchar(16));".format(common_table)) + tdLog.debug("Create common table %s" % common_table) + + # super table + for super_table in self.table_dic["super_table"]: + tdSql.execute("create stable {} (ts timestamp, c_ts timestamp, c_int int, c_bigint bigint, c_double double, c_nchar nchar(16)) tags (t1 timestamp, t2 int, t3 binary(16));".format(super_table)) + tdLog.debug("Create super table %s" % super_table) + + # child table + for i in range(len(self.table_dic["child_table"])): + if self.table_dic["child_table"][i].startswith("ct1"): + tdSql.execute("create table {} using {} tags('{}', {}, '{}');".format(self.table_dic["child_table"][i], "st1", self.table_dic["tags_value"][i][0], self.table_dic["tags_value"][i][1], self.table_dic["tags_value"][i][2])) + elif self.table_dic["child_table"][i].startswith("ct2"): + tdSql.execute("create table {} using {} tags('{}', {}, '{}');".format(self.table_dic["child_table"][i], "st2", self.table_dic["tags_value"][i][0], self.table_dic["tags_value"][i][1], self.table_dic["tags_value"][i][2])) + + # insert data + table_list = ["t1", "t2", "ct1_1", "ct1_2", "ct2_1", "ct2_2"] + for t in table_list: + sql = "insert into {} values".format(t) + for i in range(self.row_num): + sql += "({}, {}, {}, {}, {}, '{}'),".format(self.start_ts + i * 1000, self.start_ts + i * 1000, 32767+i, 65535+i, i, t + str(i)) + sql += ";" + tdSql.execute(sql) + tdLog.debug("Insert data into table %s" % t) + + def test_normal_query(self): + # only one timestamp + tdSql.query("select elapsed(ts) from t1 group by c_ts;") + assert(len(tdSql.queryResult) == self.row_num and tdSql.queryResult[0][0] == 0) + + tdSql.query("select elapsed(ts, 1m) from t1 group by c_ts;") + assert(len(tdSql.queryResult) == self.row_num and tdSql.queryResult[0][0] == 0) + + # child table with group by + tdSql.query("select elapsed(ts) from ct1_2 group by tbname;") + assert(len(tdSql.queryResult) == 1 and tdSql.queryResult[0][0] == 99000) + + # empty super table + tdSql.query("select elapsed(ts, 1s) from st_empty group by tbname;") + assert(len(tdSql.queryResult) == 0) + + # empty child table + tdSql.query("select elapsed(ts, 1s) from ct1_empty group by tbname;") + assert(len(tdSql.queryResult) == 0) + + # empty common table + tdSql.query("select elapsed(ts, 1s) from t_empty group by tbname;") + assert(len(tdSql.queryResult) == 0) + + # unit as second + tdSql.query("select elapsed(ts, 1s) from st2 group by tbname;") + assert(len(tdSql.queryResult) == 2 and tdSql.queryResult[0][0] == 99) + + # unit as minute + tdSql.query("select elapsed(ts, 1m) from st2 group by tbname;") + assert(len(tdSql.queryResult) == 2 and tdSql.queryResult[0][0] == 1.65) + + # unit as hour + tdSql.query("select elapsed(ts, 1h) from st2 group by tbname;") + assert(len(tdSql.queryResult) == 2 and tdSql.queryResult[0][0] == 0.0275) + + def test_query_with_filter(self): + end_ts = 1677654000000 + 1000 * 99 + query_list = [ + { + "sql": "select elapsed(ts, 1s) from st1 where ts >= 1677654000000 group by tbname;", + "res": [(99.0, ), (99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [(99.0, ), (99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts >= 1677654000000 and c_ts >= 1677654000000 and t1='2023-03-01 15:10:00.000' group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from st_empty where ts >= 1677654000000 and c_ts >= 1677654000000 and t1='2023-03-01 15:10:00.000' group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 where ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_2 where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_empty where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from t1 where ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from t2 where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from t_empty where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_ts > {} group by tbname;".format(end_ts), + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_ts > {} and t1='2023-03-01 15:10:00' group by tbname;".format(end_ts), + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_int < 1 group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_int >= 1 and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_int <> 1 and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar like 'ct2_%' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar like 'ct1_%' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar match '^ct2_' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar nmatch '^ct1_' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and t3 like 'g%' group by tbname;", + "res": [(99,)] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + if len(q["res"]) == 0: + assert(len(tdSql.queryResult) == len(q["res"])) + else: + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_other_function(self): + query_list = [ + { + "sql": "select avg(c_int), count(*), elapsed(ts, 1s), leastsquares(c_int, 0, 1), spread(c_bigint), sum(c_int), hyperloglog(c_int) from st1;", + "res": [(32816.5, 200, 99.0, '{slop:0.499962, intercept:32766.753731}', 99.0, 6563300, 100)] + }, + { + "sql": "select twa(c_int) * elapsed(ts, 1s) from ct1_1;", + "res": [(3.248833500000000e+06,)] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_join(self): + query_list = [ + { + "sql": "select elapsed(st1.ts, 1s) from st1, st2 where st1.ts = st2.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, st_empty where st1.ts = st_empty.ts and st1.c_ts = st_empty.c_ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, ct1_1 where st1.ts = ct1_1.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, ct1_2 ct2 where ct1.ts = ct2.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, ct1_empty ct2 where ct1.ts = ct2.ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, ct1_empty where st1.ts = ct1_empty.ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, t1 where st1.ts = t1.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, t_empty where st1.ts = t_empty.ts;", + "res": [] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, t1 t2 where ct1.ts = t2.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, t_empty t2 where ct1.ts = t2.ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, st2, st_empty where st1.ts=st2.ts and st2.ts=st_empty.ts;", + "res": [] + } + ] + + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_union(self): + query_list = [ + { + "sql": "select elapsed(ts, 1s) from st1 union select elapsed(ts, 1s) from st2;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 union all select elapsed(ts, 1s) from st2;", + "res": [(99,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 union all select elapsed(ts, 1s) from st_empty;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 union all select elapsed(ts, 1s) from ct1_2;", + "res": [(99,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 union select elapsed(ts, 1s) from ct1_2;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 union select elapsed(ts, 1s) from ct1_empty;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts < '2023-03-01 15:05:00.000' union select elapsed(ts, 1s) from ct1_1 where ts >= '2023-03-01 15:01:00.000';", + "res": [(39,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_empty union select elapsed(ts, 1s) from t_empty;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st1 group by tbname union select elapsed(ts, 1s) from st2 group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 group by tbname union all select elapsed(ts, 1s) from st2 group by tbname;", + "res": [(99,),(99,),(99,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st_empty group by tbname union all select elapsed(ts, 1s) from st2 group by tbname;", + "res": [(39,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from t1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000' interval(10s) fill(next) union select elapsed(ts, 1s) from st2 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:49.000' interval(5s) fill(prev);", + "res": [(9,), (), (4,), (5,),(10,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 group by tbname union select elapsed(ts, 1s) from st2 group by tbname union select elapsed(ts, 1s) from st_empty group by tbname;", + "res": [(99,)] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_window(self): + query_list = [ + { + "sql": "select elapsed(ts, 1s) from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' interval(10s) fill(next);", + "res": [(10,),(10,)()] + }, + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:20.000' and c_int > 100) where ts >= '2023-03-01 15:01:00.000' and ts < '2023-03-01 15:02:00.000' interval(10s) fill(prev);", + "res": [(10,)(10,)(),(),(),()] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' session(ts, 2s);", + "res": [(20,)] + }, + { + "sql": "select elapsed(ts, 1s) from st_empty where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' session(ts, 2s);", + "res": [] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_nested_query(self): + query_list = [ + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where c_int > 10 and ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000');", + "res": [(99,)] + }, + { + "sql": "select sum(v) from (select elapsed(ts, 1s) as v from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' interval(10s) fill(next));", + "res": [(20,)] + }, + { + "sql": "select avg(v) from (select elapsed(ts, 1s) as v from st2 group by tbname order by v);", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000') where c_int > 10;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where c_int > 10 and ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000') where c_int < 20;", + "res": [] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_abnormal_query(self): + # incorrect parameter + table_list = self.table_dic["super_table"] + self.table_dic["child_table"] + self.table_dic["common_table"] + incorrect_parameter_list = ["()", "(null)", "(*)", "(c_ts)", "(c_ts, 1s)", "(c_int)", "(c_bigint)", "(c_double)", "(c_nchar)", "(ts, null)", + "(ts, *)", "(2024-01-09 17:00:00)", "(2024-01-09 17:00:00, 1s)", "(t1)", "(t1, 1s)", "(t2)", "(t3)"] + for table in table_list: + for param in incorrect_parameter_list: + if table.startswith("st"): + tdSql.error("select elapsed{} from {} group by tbname order by ts;".format(param, table)) + else: + tdSql.error("select elapsed{} from {};".format(param, table)) + tdSql.error("select elapsed{} from {} group by ".format(param, table)) + + # query with unsupported function, like leastsquares、diff、derivative、top、bottom、last_row、interp + tdSql.error(" select elapsed(leastsquares(c_int, 1, 2)) from st1 group by tbname;") + tdSql.error("select elapsed(diff(ts)) from st1;") + tdSql.error("select elapsed(derivative(ts, 1s, 1)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(top(ts, 5)) from st1 group by tbname order by ts;") + tdSql.error("select top(elapsed(ts), 5) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(bottom(ts)) from st1 group by tbname order by ts;") + tdSql.error("select bottom(elapsed(ts)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(last_row(ts)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(interp(ts, 0)) from st1 group by tbname order by ts;") + + # nested aggregate function + tdSql.error("select avg(elapsed(ts, 1s)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(avg(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(sum(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(count(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(min(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(max(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(first(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(last(ts), 1s) from st1 group by tbname order by ts;") + + # other error + tdSql.error("select elapsed(ts, 1s) from t1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000' interval(10s) fill(next) union select elapsed(ts, 1s) from st2 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:49.000' interval(5s) fill(prev) group by tbname;") + tdSql.error("select elapsed(time ,1s) from (select elapsed(ts,1s) time from st1);") + tdSql.error("select elapsed(ts, 1s) from (select elapsed(ts, 1s) ts from st2);") + tdSql.error("select elapsed(time, 1s) from (select elapsed(ts, 1s) time from st1 group by tbname);") + tdSql.error("select elapsed(ts , 1s) from (select elapsed(ts, 1s) ts from st2 group by tbname);") + tdSql.error("select elapsed(ts, 1s) from (select * from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000' interval(10s) fill(next)) where c_int > 10;") + + def run(self): + self.prepareData() + self.test_normal_query() + self.test_query_with_filter() + self.test_query_with_other_function() + self.test_query_with_join() + self.test_query_with_union() + self.test_abnormal_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 694af127af..239ed7a25e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -12,6 +12,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py +,,y,army,python3 ./test.py -f community/query/function/test_fun_elapsed.py # From 366de880a85331d3c3028ab4eaeed8e9b8b4fe08 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 Jan 2024 10:48:42 +0800 Subject: [PATCH 26/81] opti:[TD-28118] raw block data for tmq --- source/libs/parser/src/parInsertUtil.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index a924ed68b0..9a9e73876d 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -662,6 +662,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate char* p = (char*)data; // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | + int32_t version = *(int32_t*)data; p += sizeof(int32_t); p += sizeof(int32_t); @@ -717,7 +718,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength) { + if (needChangeLength && version == 1) { pStart += htonl(colLength[j]); } else { pStart += colLength[j]; @@ -748,7 +749,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } fields += sizeof(int8_t) + sizeof(int32_t); - if (needChangeLength) { + if (needChangeLength && version == 1) { pStart += htonl(colLength[i]); } else { pStart += colLength[i]; From aa39a151600f46daa41c5a83e961dac38f5484ff Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 18 Jan 2024 10:42:53 +0800 Subject: [PATCH 27/81] update test cases --- .../query/function/test_fun_elapsed.py | 420 ++++++++++++++++++ tests/parallel_test/cases.task | 1 + 2 files changed, 421 insertions(+) create mode 100644 tests/army/community/query/function/test_fun_elapsed.py diff --git a/tests/army/community/query/function/test_fun_elapsed.py b/tests/army/community/query/function/test_fun_elapsed.py new file mode 100644 index 0000000000..0279e92704 --- /dev/null +++ b/tests/army/community/query/function/test_fun_elapsed.py @@ -0,0 +1,420 @@ +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.eos import * + + +class TDTestCase(TBase): + """Verify the elapsed function + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.table_dic = { + "super_table": ["st1", "st2", "st_empty"], + "child_table": ["ct1_1", "ct1_2", "ct1_empty", "ct2_1", "ct2_2", "ct2_empty"], + "tags_value": [("2023-03-01 15:00:00", 1, 'bj'), ("2023-03-01 15:10:00", 2, 'sh'), ("2023-03-01 15:20:00", 3, 'sz'), ("2023-03-01 15:00:00", 4, 'gz'), ("2023-03-01 15:10:00", 5, 'cd'), ("2023-03-01 15:20:00", 6, 'hz')], + "common_table": ["t1", "t2", "t_empty"] + } + self.start_ts = 1677654000000 # 2023-03-01 15:00:00.000 + self.row_num = 100 + + def prepareData(self): + # db + tdSql.execute("create database {};".format(self.dbname)) + tdSql.execute("use {};".format(self.dbname)) + tdLog.debug("Create database %s" % self.dbname) + + # commont table + for common_table in self.table_dic["common_table"]: + tdSql.execute("create table {} (ts timestamp, c_ts timestamp, c_int int, c_bigint bigint, c_double double, c_nchar nchar(16));".format(common_table)) + tdLog.debug("Create common table %s" % common_table) + + # super table + for super_table in self.table_dic["super_table"]: + tdSql.execute("create stable {} (ts timestamp, c_ts timestamp, c_int int, c_bigint bigint, c_double double, c_nchar nchar(16)) tags (t1 timestamp, t2 int, t3 binary(16));".format(super_table)) + tdLog.debug("Create super table %s" % super_table) + + # child table + for i in range(len(self.table_dic["child_table"])): + if self.table_dic["child_table"][i].startswith("ct1"): + tdSql.execute("create table {} using {} tags('{}', {}, '{}');".format(self.table_dic["child_table"][i], "st1", self.table_dic["tags_value"][i][0], self.table_dic["tags_value"][i][1], self.table_dic["tags_value"][i][2])) + elif self.table_dic["child_table"][i].startswith("ct2"): + tdSql.execute("create table {} using {} tags('{}', {}, '{}');".format(self.table_dic["child_table"][i], "st2", self.table_dic["tags_value"][i][0], self.table_dic["tags_value"][i][1], self.table_dic["tags_value"][i][2])) + + # insert data + table_list = ["t1", "t2", "ct1_1", "ct1_2", "ct2_1", "ct2_2"] + for t in table_list: + sql = "insert into {} values".format(t) + for i in range(self.row_num): + sql += "({}, {}, {}, {}, {}, '{}'),".format(self.start_ts + i * 1000, self.start_ts + i * 1000, 32767+i, 65535+i, i, t + str(i)) + sql += ";" + tdSql.execute(sql) + tdLog.debug("Insert data into table %s" % t) + + def test_normal_query(self): + # only one timestamp + tdSql.query("select elapsed(ts) from t1 group by c_ts;") + assert(len(tdSql.queryResult) == self.row_num and tdSql.queryResult[0][0] == 0) + + tdSql.query("select elapsed(ts, 1m) from t1 group by c_ts;") + assert(len(tdSql.queryResult) == self.row_num and tdSql.queryResult[0][0] == 0) + + # child table with group by + tdSql.query("select elapsed(ts) from ct1_2 group by tbname;") + assert(len(tdSql.queryResult) == 1 and tdSql.queryResult[0][0] == 99000) + + # empty super table + tdSql.query("select elapsed(ts, 1s) from st_empty group by tbname;") + assert(len(tdSql.queryResult) == 0) + + # empty child table + tdSql.query("select elapsed(ts, 1s) from ct1_empty group by tbname;") + assert(len(tdSql.queryResult) == 0) + + # empty common table + tdSql.query("select elapsed(ts, 1s) from t_empty group by tbname;") + assert(len(tdSql.queryResult) == 0) + + # unit as second + tdSql.query("select elapsed(ts, 1s) from st2 group by tbname;") + assert(len(tdSql.queryResult) == 2 and tdSql.queryResult[0][0] == 99) + + # unit as minute + tdSql.query("select elapsed(ts, 1m) from st2 group by tbname;") + assert(len(tdSql.queryResult) == 2 and tdSql.queryResult[0][0] == 1.65) + + # unit as hour + tdSql.query("select elapsed(ts, 1h) from st2 group by tbname;") + assert(len(tdSql.queryResult) == 2 and tdSql.queryResult[0][0] == 0.0275) + + def test_query_with_filter(self): + end_ts = 1677654000000 + 1000 * 99 + query_list = [ + { + "sql": "select elapsed(ts, 1s) from st1 where ts >= 1677654000000 group by tbname;", + "res": [(99.0, ), (99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [(99.0, ), (99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts >= 1677654000000 and c_ts >= 1677654000000 and t1='2023-03-01 15:10:00.000' group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from st_empty where ts >= 1677654000000 and c_ts >= 1677654000000 and t1='2023-03-01 15:10:00.000' group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 where ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_2 where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_empty where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from t1 where ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from t2 where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [(99.0, )] + }, + { + "sql": "select elapsed(ts, 1s) from t_empty where ts >= 1677654000000 and c_ts >= 1677654000000 group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_ts > {} group by tbname;".format(end_ts), + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_ts > {} and t1='2023-03-01 15:10:00' group by tbname;".format(end_ts), + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_int < 1 group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_int >= 1 and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_int <> 1 and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar like 'ct2_%' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar like 'ct1_%' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar match '^ct2_' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and c_nchar nmatch '^ct1_' and t1='2023-03-01 15:10:00' group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st2 where ts >= 1677654000000 and t3 like 'g%' group by tbname;", + "res": [(99,)] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + if len(q["res"]) == 0: + assert(len(tdSql.queryResult) == len(q["res"])) + else: + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_other_function(self): + query_list = [ + { + "sql": "select avg(c_int), count(*), elapsed(ts, 1s), leastsquares(c_int, 0, 1), spread(c_bigint), sum(c_int), hyperloglog(c_int) from st1;", + "res": [(32816.5, 200, 99.0, '{slop:0.499962, intercept:32766.753731}', 99.0, 6563300, 100)] + }, + { + "sql": "select twa(c_int) * elapsed(ts, 1s) from ct1_1;", + "res": [(3.248833500000000e+06,)] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_join(self): + query_list = [ + { + "sql": "select elapsed(st1.ts, 1s) from st1, st2 where st1.ts = st2.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, st_empty where st1.ts = st_empty.ts and st1.c_ts = st_empty.c_ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, ct1_1 where st1.ts = ct1_1.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, ct1_2 ct2 where ct1.ts = ct2.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, ct1_empty ct2 where ct1.ts = ct2.ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, ct1_empty where st1.ts = ct1_empty.ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, t1 where st1.ts = t1.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, t_empty where st1.ts = t_empty.ts;", + "res": [] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, t1 t2 where ct1.ts = t2.ts;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ct1.ts, 1s) from ct1_1 ct1, t_empty t2 where ct1.ts = t2.ts;", + "res": [] + }, + { + "sql": "select elapsed(st1.ts, 1s) from st1, st2, st_empty where st1.ts=st2.ts and st2.ts=st_empty.ts;", + "res": [] + } + ] + + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_union(self): + query_list = [ + { + "sql": "select elapsed(ts, 1s) from st1 union select elapsed(ts, 1s) from st2;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 union all select elapsed(ts, 1s) from st2;", + "res": [(99,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 union all select elapsed(ts, 1s) from st_empty;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 union all select elapsed(ts, 1s) from ct1_2;", + "res": [(99,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 union select elapsed(ts, 1s) from ct1_2;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_1 union select elapsed(ts, 1s) from ct1_empty;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts < '2023-03-01 15:05:00.000' union select elapsed(ts, 1s) from ct1_1 where ts >= '2023-03-01 15:01:00.000';", + "res": [(39,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from ct1_empty union select elapsed(ts, 1s) from t_empty;", + "res": [] + }, + { + "sql": "select elapsed(ts, 1s) from st1 group by tbname union select elapsed(ts, 1s) from st2 group by tbname;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 group by tbname union all select elapsed(ts, 1s) from st2 group by tbname;", + "res": [(99,),(99,),(99,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from st_empty group by tbname union all select elapsed(ts, 1s) from st2 group by tbname;", + "res": [(99,),(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from t1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000' interval(10s) fill(next) union select elapsed(ts, 1s) from st2 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:49.000' interval(5s) fill(prev);", + "res": [(9,), (None,), (4,), (5,),(10,)] + }, + { + "sql": "select elapsed(ts, 1s) from st1 group by tbname union select elapsed(ts, 1s) from st2 group by tbname union select elapsed(ts, 1s) from st_empty group by tbname;", + "res": [(99,)] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + tdLog.debug(q["sql"] + " with res: " + str(tdSql.queryResult)) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_query_with_window(self): + query_list = [ + { + "sql": "select elapsed(ts, 1s) from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' interval(10s) fill(next);", + "res": [(10,),(10,)()] + }, + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:20.000' and c_int > 100) where ts >= '2023-03-01 15:01:00.000' and ts < '2023-03-01 15:02:00.000' interval(10s) fill(prev);", + "res": [(10,)(10,)(),(),(),()] + }, + { + "sql": "select elapsed(ts, 1s) from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' session(ts, 2s);", + "res": [(20,)] + }, + { + "sql": "select elapsed(ts, 1s) from st_empty where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' session(ts, 2s);", + "res": [] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_nested_query(self): + query_list = [ + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where c_int > 10 and ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000');", + "res": [(99,)] + }, + { + "sql": "select sum(v) from (select elapsed(ts, 1s) as v from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:00:20.000' interval(10s) fill(next));", + "res": [(20,)] + }, + { + "sql": "select avg(v) from (select elapsed(ts, 1s) as v from st2 group by tbname order by v);", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000') where c_int > 10;", + "res": [(99,)] + }, + { + "sql": "select elapsed(ts, 1s) from (select * from st1 where c_int > 10 and ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000') where c_int < 20;", + "res": [] + } + ] + for q in query_list: + tdSql.query(q["sql"]) + tdLog.debug(q["sql"] + " with res: " + str(tdSql.queryResult)) + assert(len(tdSql.queryResult) == len(q["res"]) and tdSql.queryResult == q["res"]) + + def test_abnormal_query(self): + # incorrect parameter + table_list = self.table_dic["super_table"] + self.table_dic["child_table"] + self.table_dic["common_table"] + incorrect_parameter_list = ["()", "(null)", "(*)", "(c_ts)", "(c_ts, 1s)", "(c_int)", "(c_bigint)", "(c_double)", "(c_nchar)", "(ts, null)", + "(ts, *)", "(2024-01-09 17:00:00)", "(2024-01-09 17:00:00, 1s)", "(t1)", "(t1, 1s)", "(t2)", "(t3)"] + for table in table_list: + for param in incorrect_parameter_list: + if table.startswith("st"): + tdSql.error("select elapsed{} from {} group by tbname order by ts;".format(param, table)) + else: + tdSql.error("select elapsed{} from {};".format(param, table)) + tdSql.error("select elapsed{} from {} group by ".format(param, table)) + + # query with unsupported function, like leastsquares、diff、derivative、top、bottom、last_row、interp + tdSql.error(" select elapsed(leastsquares(c_int, 1, 2)) from st1 group by tbname;") + tdSql.error("select elapsed(diff(ts)) from st1;") + tdSql.error("select elapsed(derivative(ts, 1s, 1)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(top(ts, 5)) from st1 group by tbname order by ts;") + tdSql.error("select top(elapsed(ts), 5) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(bottom(ts)) from st1 group by tbname order by ts;") + tdSql.error("select bottom(elapsed(ts)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(last_row(ts)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(interp(ts, 0)) from st1 group by tbname order by ts;") + + # nested aggregate function + tdSql.error("select avg(elapsed(ts, 1s)) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(avg(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(sum(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(count(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(min(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(max(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(first(ts), 1s) from st1 group by tbname order by ts;") + tdSql.error("select elapsed(last(ts), 1s) from st1 group by tbname order by ts;") + + # other error + tdSql.error("select elapsed(ts, 1s) from t1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000' interval(10s) fill(next) union select elapsed(ts, 1s) from st2 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:49.000' interval(5s) fill(prev) group by tbname;") + tdSql.error("select elapsed(time ,1s) from (select elapsed(ts,1s) time from st1);") + tdSql.error("select elapsed(ts, 1s) from (select elapsed(ts, 1s) ts from st2);") + tdSql.error("select elapsed(time, 1s) from (select elapsed(ts, 1s) time from st1 group by tbname);") + tdSql.error("select elapsed(ts , 1s) from (select elapsed(ts, 1s) ts from st2 group by tbname);") + tdSql.error("select elapsed(ts, 1s) from (select * from st1 where ts between '2023-03-01 15:00:00.000' and '2023-03-01 15:01:40.000' interval(10s) fill(next)) where c_int > 10;") + + def run(self): + self.prepareData() + self.test_normal_query() + self.test_query_with_filter() + self.test_query_with_other_function() + self.test_query_with_join() + self.test_query_with_union() + self.test_abnormal_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 694af127af..239ed7a25e 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -12,6 +12,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py +,,y,army,python3 ./test.py -f community/query/function/test_fun_elapsed.py # From 1b85adaa3ae3d7889c3704d3407970f6038d27b7 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 18 Jan 2024 14:17:08 +0800 Subject: [PATCH 28/81] opti:[TD-28118] raw block data for tmq --- source/client/src/clientTmq.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index d98c3340c1..69681b9ae0 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1577,16 +1577,19 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - // extract the rows in this data packet + bool needTransformSchema = !pRspObj->rsp.withSchema; + if (!pRspObj->rsp.withSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable + pRspObj->rsp.withSchema = true; + pRspObj->rsp.blockSchema = taosArrayInit(pRspObj->rsp.blockNum, sizeof(void*)); + } + // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); int64_t rows = htobe64(pRetrieve->numOfRows); pVg->numOfRows += rows; (*numOfRows) += rows; - if (!pRspObj->rsp.withSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable - pRspObj->rsp.withSchema = true; - pRspObj->rsp.blockSchema = taosArrayInit(pRspObj->rsp.blockNum, sizeof(void *)); + if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); if(schema){ taosArrayPush(pRspObj->rsp.blockSchema, &schema); From 00c8405fc95a48a125e0a62337f37ddd9d0b6d6c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 18 Jan 2024 15:10:24 +0800 Subject: [PATCH 29/81] fix: add query_basic.py to task --- tests/army/community/query/query_basic.py | 7 +- tests/army/frame/sql.py | 314 ++++++++++++---------- tests/parallel_test/cases.task | 1 + 3 files changed, 173 insertions(+), 149 deletions(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index e3fdeb0f34..159a70e0d1 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -52,13 +52,14 @@ class TDTestCase(TBase): tdLog.info(f"do query.") # __group_key - sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti;" + sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti" tdSql.execute(sql) - tdSql.checkRows(251) + # column index 1 value same with 2 + tdSql.checkSameColumn(1, 2) sql = f"select count(*),_group_key(usi) from {self.stb} group by usi;" tdSql.execute(sql) - tdSql.checkRows(997) + tdSql.checkSameColumn(1, 2) # tail sql1 = "select ts,ui from d0 order by ts desc limit 5 offset 2;" diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 5477f96b3d..a513edfb13 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -78,6 +78,107 @@ class TDSql: self.cursor.execute(s) time.sleep(2) + + # + # do execute + # + + def query(self, sql, row_tag=None, queryTimes=10, count_expected_res=None): + self.sql = sql + i=1 + while i <= queryTimes: + try: + self.cursor.execute(sql) + self.res = self.cursor.fetchall() + self.queryRows = len(self.res) + self.queryCols = len(self.cursor.description) + + if count_expected_res is not None: + counter = 0 + while count_expected_res != self.res[0][0]: + self.cursor.execute(sql) + self.res = self.cursor.fetchall() + if counter < queryTimes: + counter += 0.5 + time.sleep(0.5) + else: + return False + if row_tag: + return self.res + return self.queryRows + except Exception as e: + tdLog.notice("Try to query again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 + time.sleep(1) + pass + + def executeTimes(self, sql, times): + for i in range(times): + try: + return self.cursor.execute(sql) + except BaseException: + time.sleep(1) + continue + + def execute(self, sql, queryTimes=30, show=False): + self.sql = sql + if show: + tdLog.info(sql) + i=1 + while i <= queryTimes: + try: + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + except Exception as e: + tdLog.notice("Try to execute sql again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 + time.sleep(1) + pass + + # execute many sql + def executes(self, sqls, queryTimes=30, show=False): + for sql in sqls: + self.execute(sql, queryTimes, show) + + def waitedQuery(self, sql, expectRows, timeout): + tdLog.info("sql: %s, try to retrieve %d rows in %d seconds" % (sql, expectRows, timeout)) + self.sql = sql + try: + for i in range(timeout): + self.cursor.execute(sql) + self.res = self.cursor.fetchall() + self.queryRows = len(self.res) + self.queryCols = len(self.cursor.description) + tdLog.info("sql: %s, try to retrieve %d rows,get %d rows" % (sql, expectRows, self.queryRows)) + if self.queryRows >= expectRows: + return (self.queryRows, i) + time.sleep(1) + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + return (self.queryRows, timeout) + + def is_err_sql(self, sql): + err_flag = True + try: + self.cursor.execute(sql) + except BaseException: + err_flag = False + + return False if err_flag else True + def error(self, sql, expectedErrno = None, expectErrInfo = None): caller = inspect.getframeinfo(inspect.stack()[1][0]) expectErrNotOccured = True @@ -95,7 +196,7 @@ class TDSql: else: self.queryRows = 0 self.queryCols = 0 - self.queryResult = None + self.res = None if expectedErrno != None: if expectedErrno == self.errno: @@ -115,49 +216,25 @@ class TDSql: return self.error_info - def query(self, sql, row_tag=None, queryTimes=10, count_expected_res=None): + # + # get session + # + + def getData(self, row, col): + self.checkRowCol(row, col) + return self.res[row][col] + + def getResult(self, sql): self.sql = sql - i=1 - while i <= queryTimes: - try: - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) - - if count_expected_res is not None: - counter = 0 - while count_expected_res != self.queryResult[0][0]: - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - if counter < queryTimes: - counter += 0.5 - time.sleep(0.5) - else: - return False - if row_tag: - return self.queryResult - return self.queryRows - except Exception as e: - tdLog.notice("Try to query again, query times: %d "%i) - if i == queryTimes: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - i+=1 - time.sleep(1) - pass - - - def is_err_sql(self, sql): - err_flag = True try: self.cursor.execute(sql) - except BaseException: - err_flag = False - - return False if err_flag else True + self.res = self.cursor.fetchall() + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + return self.res def getVariable(self, search_attr): ''' @@ -193,29 +270,19 @@ class TDSql: return col_name_list, col_type_list return col_name_list - def waitedQuery(self, sql, expectRows, timeout): - tdLog.info("sql: %s, try to retrieve %d rows in %d seconds" % (sql, expectRows, timeout)) - self.sql = sql - try: - for i in range(timeout): - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) - tdLog.info("sql: %s, try to retrieve %d rows,get %d rows" % (sql, expectRows, self.queryRows)) - if self.queryRows >= expectRows: - return (self.queryRows, i) - time.sleep(1) - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - return (self.queryRows, timeout) - def getRows(self): return self.queryRows + # get first value + def getFirstValue(self, sql) : + self.query(sql) + return self.getData(0, 0) + + + # + # check session + # + def checkRows(self, expectRows): if self.queryRows == expectRows: tdLog.info("sql:%s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) @@ -273,26 +340,26 @@ class TDSql: self.checkRowCol(row, col) - if self.queryResult[row][col] != data: + if self.res[row][col] != data: if self.cursor.istype(col, "TIMESTAMP"): # suppose user want to check nanosecond timestamp if a longer data passed`` if isinstance(data,str) : if (len(data) >= 28): - if self.queryResult[row][col] == _parse_ns_timestamp(data): + if self.res[row][col] == _parse_ns_timestamp(data): if(show): tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) else: - if self.queryResult[row][col].astimezone(datetime.timezone.utc) == _parse_datetime(data).astimezone(datetime.timezone.utc): - # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + if self.res[row][col].astimezone(datetime.timezone.utc) == _parse_datetime(data).astimezone(datetime.timezone.utc): + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}") if(show): tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) return elif isinstance(data,int): @@ -304,72 +371,72 @@ class TDSql: precision = 'ns' else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) return success = False if precision == 'ms': - dt_obj = self.queryResult[row][col] + dt_obj = self.res[row][col] ts = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1000) + int(dt_obj.microsecond/1000) if ts == data: success = True elif precision == 'us': - dt_obj = self.queryResult[row][col] + dt_obj = self.res[row][col] ts = int(int((dt_obj-datetime.datetime.fromtimestamp(0,dt_obj.tzinfo)).total_seconds())*1e6) + int(dt_obj.microsecond) if ts == data: success = True elif precision == 'ns': - if data == self.queryResult[row][col]: + if data == self.res[row][col]: success = True if success: if(show): tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) return elif isinstance(data,datetime.datetime): - dt_obj = self.queryResult[row][col] + dt_obj = self.res[row][col] delt_data = data-datetime.datetime.fromtimestamp(0,data.tzinfo) - delt_result = self.queryResult[row][col] - datetime.datetime.fromtimestamp(0,self.queryResult[row][col].tzinfo) + delt_result = self.res[row][col] - datetime.datetime.fromtimestamp(0,self.res[row][col].tzinfo) if delt_data == delt_result: if(show): tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) return else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) - if str(self.queryResult[row][col]) == str(data): - # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + if str(self.res[row][col]) == str(data): + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}") if(show): tdLog.info("check successfully") return elif isinstance(data, float): - if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001: - # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + if abs(data) >= 1 and abs((self.res[row][col] - data) / data) <= 0.000001: + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}") if(show): tdLog.info("check successfully") - elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001: - # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + elif abs(data) < 1 and abs(self.res[row][col] - data) <= 0.000001: + # tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.res[row][col]} == expect:{data}") if(show): tdLog.info("check successfully") else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) return else: caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + args = (caller.filename, caller.lineno, self.sql, row, col, self.res[row][col], data) tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) if(show): tdLog.info("check successfully") @@ -397,23 +464,23 @@ class TDSql: def checkDataNoExit(self, row, col, data): if self.checkRowColNoExit(row, col) == False: return False - if self.queryResult[row][col] != data: + if self.res[row][col] != data: if self.cursor.istype(col, "TIMESTAMP"): # suppose user want to check nanosecond timestamp if a longer data passed if (len(data) >= 28): - if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data): + if pd.to_datetime(self.res[row][col]) == pd.to_datetime(data): return True else: - if self.queryResult[row][col] == _parse_datetime(data): + if self.res[row][col] == _parse_datetime(data): return True return False - if str(self.queryResult[row][col]) == str(data): + if str(self.res[row][col]) == str(data): return True elif isinstance(data, float): - if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001: + if abs(data) >= 1 and abs((self.res[row][col] - data) / data) <= 0.000001: return True - elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001: + elif abs(data) < 1 and abs(self.res[row][col] - data) <= 0.000001: return True else: return False @@ -437,56 +504,6 @@ class TDSql: self.query(sql) self.checkData(row, col, data) - - def getData(self, row, col): - self.checkRowCol(row, col) - return self.queryResult[row][col] - - def getResult(self, sql): - self.sql = sql - try: - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - return self.queryResult - - def executeTimes(self, sql, times): - for i in range(times): - try: - return self.cursor.execute(sql) - except BaseException: - time.sleep(1) - continue - - def execute(self, sql, queryTimes=30, show=False): - self.sql = sql - if show: - tdLog.info(sql) - i=1 - while i <= queryTimes: - try: - self.affectedRows = self.cursor.execute(sql) - return self.affectedRows - except Exception as e: - tdLog.notice("Try to execute sql again, query times: %d "%i) - if i == queryTimes: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - i+=1 - time.sleep(1) - pass - - # execute many sql - def executes(self, sqls, queryTimes=30, show=False): - for sql in sqls: - self.execute(sql, queryTimes, show) - def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: caller = inspect.getframeinfo(inspect.stack()[1][0]) @@ -544,17 +561,22 @@ class TDSql: def checkAgg(self, sql, expectCnt): self.query(sql) self.checkData(0, 0, expectCnt) - - # get first value - def getFirstValue(self, sql) : - self.query(sql) - return self.getData(0, 0) # expect first value def checkFirstValue(self, sql, expect): self.query(sql) self.checkData(0, 0, expect) - + + # colIdx1 value same with colIdx2 + def checkSameColumn(self, c1, c2): + for i in range(self.queryRows): + if self.res[i][c1] != self.res[i][c2]: + tdLog.exit(f"Not same. row={i} col1={c1} col2={c2}. {self.res[i][c1]}!={self.res[i][c2]}") + tdLog.info(f"check {self.queryRows} rows. column{c1} value equal tp column{c2}") + + # + # others session + # def get_times(self, time_str, precision="ms"): caller = inspect.getframeinfo(inspect.stack()[1][0]) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 694af127af..b0e4b5725a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,6 +11,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py From f7d7ce35155c8b0159f5dc2f3195c90813957d1d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 18 Jan 2024 15:33:07 +0800 Subject: [PATCH 30/81] fix: rename all queryResult name --- tests/army/frame/caseBase.py | 4 ++-- tests/army/frame/common.py | 28 ++++++++++++++-------------- tests/army/frame/server/cluster.py | 3 +-- tests/army/test.py | 11 ----------- 4 files changed, 17 insertions(+), 29 deletions(-) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index fcfcb7d066..b1500b69e1 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -193,10 +193,10 @@ class TBase: # sql rows1 = tdSql.query(sql1,queryTimes=2) - res1 = copy.deepcopy(tdSql.queryResult) + res1 = copy.deepcopy(tdSql.res) tdSql.query(sql2,queryTimes=2) - res2 = tdSql.queryResult + res2 = tdSql.res rowlen1 = len(res1) rowlen2 = len(res2) diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index 5cdb3f9f46..d11a0dbb4d 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -795,7 +795,7 @@ class TDCom: def getOneRow(self, location, containElm): res_list = list() if 0 <= location < tdSql.queryRows: - for row in tdSql.queryResult: + for row in tdSql.res: if row[location] == containElm: res_list.append(row) return res_list @@ -943,7 +943,7 @@ class TDCom: """drop all streams """ tdSql.query("show streams") - stream_name_list = list(map(lambda x: x[0], tdSql.queryResult)) + stream_name_list = list(map(lambda x: x[0], tdSql.res)) for stream_name in stream_name_list: tdSql.execute(f'drop stream if exists {stream_name};') @@ -962,7 +962,7 @@ class TDCom: """drop all databases """ tdSql.query("show databases;") - db_list = list(map(lambda x: x[0], tdSql.queryResult)) + db_list = list(map(lambda x: x[0], tdSql.res)) for dbname in db_list: if dbname not in self.white_list and "telegraf" not in dbname: tdSql.execute(f'drop database if exists `{dbname}`') @@ -1412,7 +1412,7 @@ class TDCom: input_function (str): scalar """ tdSql.query(sql) - res = tdSql.queryResult + res = tdSql.res if input_function in ["acos", "asin", "atan", "cos", "log", "pow", "sin", "sqrt", "tan"]: tdSql.checkEqual(res[1][1], "DOUBLE") tdSql.checkEqual(res[2][1], "DOUBLE") @@ -1490,7 +1490,7 @@ class TDCom: bigint: bigint-ts """ tdSql.query(f'select cast({str_ts} as bigint)') - return tdSql.queryResult[0][0] + return tdSql.res[0][0] def cast_query_data(self, query_data): """cast query-result for existed-stb @@ -1514,7 +1514,7 @@ class TDCom: tdSql.query(f'select cast("{v}" as binary(6))') else: tdSql.query(f'select cast("{v}" as {" ".join(col_tag_type_list[i].strip().split(" ")[1:])})') - query_data_l[i] = tdSql.queryResult[0][0] + query_data_l[i] = tdSql.res[0][0] else: query_data_l[i] = v nl.append(tuple(query_data_l)) @@ -1566,9 +1566,9 @@ class TDCom: if tag_value_list: dvalue = len(self.tag_type_str.split(',')) - defined_tag_count tdSql.query(sql1) - res1 = tdSql.queryResult + res1 = tdSql.res tdSql.query(sql2) - res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + res2 = self.cast_query_data(tdSql.res) if tag_value_list or use_exist_stb else tdSql.res tdSql.sql = sql1 new_list = list() if tag_value_list: @@ -1601,10 +1601,10 @@ class TDCom: tdLog.info("query retrying ...") new_list = list() tdSql.query(sql1) - res1 = tdSql.queryResult + res1 = tdSql.res tdSql.query(sql2) - # res2 = tdSql.queryResult - res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + # res2 = tdSql.res + res2 = self.cast_query_data(tdSql.res) if tag_value_list or use_exist_stb else tdSql.res tdSql.sql = sql1 if tag_value_list: @@ -1643,10 +1643,10 @@ class TDCom: tdLog.info("query retrying ...") new_list = list() tdSql.query(sql1) - res1 = tdSql.queryResult + res1 = tdSql.res tdSql.query(sql2) - # res2 = tdSql.queryResult - res2 = self.cast_query_data(tdSql.queryResult) if tag_value_list or use_exist_stb else tdSql.queryResult + # res2 = tdSql.res + res2 = self.cast_query_data(tdSql.res) if tag_value_list or use_exist_stb else tdSql.res tdSql.sql = sql1 if tag_value_list: diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py index ade8ac39a2..aba2d2e630 100644 --- a/tests/army/frame/server/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -86,10 +86,9 @@ class ConfigureyCluster: count=0 while count < 5: tdSql.query("select * from information_schema.ins_dnodes") - # tdLog.debug(tdSql.queryResult) status=0 for i in range(self.dnodeNums): - if tdSql.queryResult[i][4] == "ready": + if tdSql.res[i][4] == "ready": status+=1 # tdLog.debug(status) diff --git a/tests/army/test.py b/tests/army/test.py index a3e28b772d..a94fefe0be 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -560,17 +560,6 @@ if __name__ == "__main__": conn = taosws.connect(f"taosws://root:taosdata@{host}:6041") else: conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) - # tdSql.init(conn.cursor()) - # tdSql.execute("create qnode on dnode 1") - # tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) - # tdSql.query("show local variables;") - # for i in range(tdSql.queryRows): - # if tdSql.queryResult[i][0] == "queryPolicy" : - # if int(tdSql.queryResult[i][1]) == int(queryPolicy): - # tdLog.info('alter queryPolicy to %d successfully'%queryPolicy) - # else : - # tdLog.debug(tdSql.queryResult) - # tdLog.exit("alter queryPolicy to %d failed"%queryPolicy) cursor = conn.cursor() cursor.execute("create qnode on dnode 1") From be3f16b7e4c16077cdf6389ad43a239a4af925b6 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Thu, 18 Jan 2024 15:53:16 +0800 Subject: [PATCH 31/81] fix: taos -n server killed --- tests/army/community/cmdline/fullopt.py | 2 ++ tests/army/community/query/query_basic.py | 6 +++--- tests/army/frame/sql.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/army/community/cmdline/fullopt.py b/tests/army/community/cmdline/fullopt.py index 99c7095a38..39d1d581ed 100644 --- a/tests/army/community/cmdline/fullopt.py +++ b/tests/army/community/cmdline/fullopt.py @@ -119,6 +119,8 @@ class TDTestCase(TBase): # stop taosd test taos as server sc.dnodeStop(idx) etool.exeBinFile("taos", f'-n server', wait=False) + time.sleep(3) + eos.exe("pkill -9 taos") # run def run(self): diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 159a70e0d1..90916a1c6c 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -53,12 +53,12 @@ class TDTestCase(TBase): # __group_key sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti" - tdSql.execute(sql) + tdSql.query(sql) # column index 1 value same with 2 tdSql.checkSameColumn(1, 2) - sql = f"select count(*),_group_key(usi) from {self.stb} group by usi;" - tdSql.execute(sql) + sql = f"select count(*),_group_key(usi),usi from {self.stb} group by usi limit 100;" + tdSql.query(sql) tdSql.checkSameColumn(1, 2) # tail diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index a513edfb13..8b7538321d 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -572,7 +572,7 @@ class TDSql: for i in range(self.queryRows): if self.res[i][c1] != self.res[i][c2]: tdLog.exit(f"Not same. row={i} col1={c1} col2={c2}. {self.res[i][c1]}!={self.res[i][c2]}") - tdLog.info(f"check {self.queryRows} rows. column{c1} value equal tp column{c2}") + tdLog.info(f"check {self.queryRows} rows two column value same. column index [{c1},{c2}]") # # others session From 64858af92f0b60d38665cb645a97366f5f27ac77 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 Jan 2024 16:30:18 +0800 Subject: [PATCH 32/81] fix: calculate interval end with new alogrithm --- source/libs/executor/src/executil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1da7818c3c..0df4241c5d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1862,7 +1862,7 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI STimeWindow w = {0}; if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value getInitialStartTimeWindow(pInterval, ts, &w, (order == TSDB_ORDER_ASC)); - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + w.ekey = taosTimeGetIntervalEnd(w.skey, pInterval); return w; } From bb715c249d770c2160d7058ab3b5b22ac5e89a13 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Thu, 18 Jan 2024 16:58:55 +0800 Subject: [PATCH 33/81] delete duplicated cases --- tests/system-test/2-query/orderBy.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/system-test/2-query/orderBy.py b/tests/system-test/2-query/orderBy.py index dedc57eab3..1cf1478439 100644 --- a/tests/system-test/2-query/orderBy.py +++ b/tests/system-test/2-query/orderBy.py @@ -278,19 +278,19 @@ class TDTestCase: def queryOrderByAgg(self): - tdSql.query("SELECT COUNT(*) FROM t1 order by COUNT(*)") + tdSql.no_error("SELECT COUNT(*) FROM t1 order by COUNT(*)") - tdSql.query("SELECT COUNT(*) FROM t1 order by last(c2)") + tdSql.no_error("SELECT COUNT(*) FROM t1 order by last(c2)") - tdSql.query("SELECT c1 FROM t1 order by last(ts)") + tdSql.no_error("SELECT c1 FROM t1 order by last(ts)") - tdSql.query("SELECT ts FROM t1 order by last(ts)") + tdSql.no_error("SELECT ts FROM t1 order by last(ts)") - tdSql.query("SELECT last(ts), ts, c1 FROM t1 order by 2") + tdSql.no_error("SELECT last(ts), ts, c1 FROM t1 order by 2") - tdSql.query("SELECT ts, last(ts) FROM t1 order by last(ts)") + tdSql.no_error("SELECT ts, last(ts) FROM t1 order by last(ts)") - tdSql.query(f"SELECT * FROM t1 order by last(ts)") + tdSql.no_error(f"SELECT * FROM t1 order by last(ts)") tdSql.query(f"SELECT last(ts) as t2, ts FROM t1 order by 1") tdSql.checkRows(1) @@ -302,6 +302,18 @@ class TDTestCase: tdSql.error(f"SELECT last(ts) as t2, ts FROM t1 order by last(t2)") + def queryOrderByAmbiguousName(self): + tdSql.error(sql="select c1 as name, c2 as name, c3 from t1 order by name", expectErrInfo='ambiguous', + fullMatched=False) + + tdSql.error(sql="select c1, c2 as c1, c3 from t1 order by c1", expectErrInfo='ambiguous', fullMatched=False) + + tdSql.error(sql='select last(ts), last(c1) as name ,last(c2) as name,last(c3) from t1 order by name', + expectErrInfo='ambiguous', fullMatched=False) + + tdSql.no_error("select c1 as name, c2 as c1, c3 from t1 order by c1") + + tdSql.no_error('select c1 as name from (select c1, c2 as name from st) order by name') # run def run(self): @@ -317,6 +329,8 @@ class TDTestCase: # agg self.queryOrderByAgg() + # td-28332 + self.queryOrderByAmbiguousName() # stop def stop(self): From b83bb8c1507d29bf4e0ef7a2a83e451ec8aa7d16 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 18 Jan 2024 17:20:31 +0800 Subject: [PATCH 34/81] fix: last_row error --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- tests/system-test/2-query/last_row.py | 46 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d86219542f..cc0bf2b774 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -941,7 +941,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr } if(lastrowTmpIndexArray != NULL) { - mergeLastCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds); + mergeLastRowCid(uid, pTsdb, &lastrowTmpColArray, pr, lastrowColIds, lastrowIndex, lastrowSlotIds); for(int i = 0; i < taosArrayGetSize(lastrowTmpColArray); i++) { taosArrayInsert(pTmpColArray, *(int32_t*)taosArrayGet(lastrowTmpIndexArray, i), taosArrayGet(lastrowTmpColArray, i)); } diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index 6469b3ea54..a7223db1cd 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -861,11 +861,55 @@ class TDTestCase: self.support_super_table_test() + def initLastRowDelayTest(self, dbname="db"): + tdSql.execute(f"drop database if exists {dbname} ") + create_db_sql = f"create database if not exists {dbname} keep 3650 duration 1000 cachemodel 'NONE' REPLICA 1" + tdSql.execute(create_db_sql) + + tdSql.execute(f"use {dbname}") + tdSql.execute(f'create stable {dbname}.st(ts timestamp, v_int int, v_float float) TAGS (ctname varchar(32))') + + tdSql.execute(f"create table {dbname}.ct1 using st tags('ct1')") + tdSql.execute(f"create table {dbname}.ct2 using st tags('ct2')") + + tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000000000,86,86)") + tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000021255,59,59)") + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'select last(*) from {dbname}.st') + tdSql.execute(f'select last_row(*) from {dbname}.st') + tdSql.execute(f"insert into {dbname}.st(tbname,ts) values('ct1',1630000091255)") + tdSql.execute(f'flush database {dbname}') + tdSql.execute(f'select last(*) from {dbname}.st') + tdSql.execute(f'select last_row(*) from {dbname}.st') + tdSql.execute(f'alter database {dbname} cachemodel "both"') + tdSql.query(f'select last(*) from {dbname}.st') + tdSql.checkData(0 , 1 , 59) + + tdSql.query(f'select last_row(*) from {dbname}.st') + tdSql.checkData(0 , 1 , None) + tdSql.checkData(0 , 2 , None) + + tdLog.printNoPrefix("========== delay test init success ==============") + + def lastRowDelayTest(self, dbname="db"): + tdLog.printNoPrefix("========== delay test start ==============") + + tdSql.execute(f"use {dbname}") + + tdSql.query(f'select last(*) from {dbname}.st') + tdSql.checkData(0 , 1 , 59) + + tdSql.query(f'select last_row(*) from {dbname}.st') + tdSql.checkData(0 , 1 , None) + tdSql.checkData(0 , 2 , None) + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring # tdSql.prepare() tdLog.printNoPrefix("==========step1:create table ==============") + self.initLastRowDelayTest("DELAYTEST") + # cache_last 0 self.prepare_datas("'NONE' ") self.prepare_tag_datas("'NONE'") @@ -890,6 +934,8 @@ class TDTestCase: self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step,"'BOTH'") self.basic_query() + self.lastRowDelayTest("DELAYTEST") + def stop(self): tdSql.close() From cd6026c6a9f34b3b603244355f8e677b0cbf490d Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 18 Jan 2024 17:31:23 +0800 Subject: [PATCH 35/81] docs: keepColumnName notes --- docs/en/14-reference/12-config/index.md | 7 ++++--- docs/zh/14-reference/12-config/index.md | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 65c48f9190..c1abfd3e39 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -226,9 +226,10 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo | Attribute | Description | | ------------- | --------------------------------------------------------------------------------------------------------------- | | Applicable | Client only | -| Meaning | When the Last, First, LastRow function is queried, whether the returned column name contains the function name. | -| Value Range | 0 means including the function name, 1 means not including the function name. | -| Default Value | 0 | +| Meaning | When the Last, First, and LastRow functions are queried and no alias is specified, the alias is automatically set to the column name (excluding the function name). Therefore, if the order by clause refers to the column name, it will automatically refer to the function corresponding to the column. | +| Value Range | 1 means automatically setting the alias to the column name (excluding the function name), 0 means not automatically setting the alias. | +| Default Value | 0 | +| Notes | When multiple of the above functions act on the same column at the same time and no alias is specified, if the order by clause refers to the column name, column selection ambiguous will occur because the aliases of multiple columns are the same. | ## Locale Parameters diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 27a9618107..4d47f0771c 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -215,9 +215,10 @@ taos -C | 属性 | 说明 | | -------- | ----------------------------------------------------------- | | 适用范围 | 仅客户端适用 | -| 含义 | Last、First、LastRow 函数查询时，返回的列名是否包含函数名。 | -| 取值范围 | 0 表示包含函数名，1 表示不包含函数名。 | -| 缺省值 | 0 | +| 含义 | Last、First、LastRow 函数查询且未指定别名时，自动设置别名为列名（不含函数名），因此 order by 子句如果引用了该列名将自动引用该列对应的函数 | +| 取值范围 | 1 表示自动设置别名为列名(不包含函数名), 0 表示不自动设置别名。 | +| 缺省值 | 0 | +| 补充说明 | 当同时出现多个上述函数作用于同一列且未指定别名时，如果 order by 子句引用了该列名，将会因为多列别名相同引发列选择冲突| ### countAlwaysReturnValue From afbe4d99a8f6ad0517907c233b51f0860578b244 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 18 Jan 2024 18:16:18 +0800 Subject: [PATCH 36/81] fix: case --- tests/system-test/2-query/last_row.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index a7223db1cd..0744b3bae5 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -866,11 +866,12 @@ class TDTestCase: create_db_sql = f"create database if not exists {dbname} keep 3650 duration 1000 cachemodel 'NONE' REPLICA 1" tdSql.execute(create_db_sql) + time.sleep(3) tdSql.execute(f"use {dbname}") tdSql.execute(f'create stable {dbname}.st(ts timestamp, v_int int, v_float float) TAGS (ctname varchar(32))') - tdSql.execute(f"create table {dbname}.ct1 using st tags('ct1')") - tdSql.execute(f"create table {dbname}.ct2 using st tags('ct2')") + tdSql.execute(f"create table {dbname}.ct1 using {dbname}.st tags('ct1')") + tdSql.execute(f"create table {dbname}.ct2 using {dbname}.st tags('ct2')") tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000000000,86,86)") tdSql.execute(f"insert into {dbname}.st(tbname,ts,v_float, v_int) values('ct1',1630000021255,59,59)") From ff8908dedebfffc72cb798a7f4a46de496d52bdd Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 18 Jan 2024 18:30:21 +0800 Subject: [PATCH 37/81] ignore invalid state --- source/libs/stream/src/streamSessionState.c | 4 ++++ source/libs/stream/src/streamState.c | 1 + tests/script/tsim/stream/partitionby1.sim | 2 ++ 3 files changed, 7 insertions(+) diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 2cb77c60dc..1f991d309f 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -90,6 +90,7 @@ SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKe SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; + pNewPos->beFlushed = true; memcpy(pNewPos->pRowBuff, p, *pVLen); taosMemoryFree(p); return pNewPos; @@ -217,6 +218,7 @@ int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, v SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; + pNewPos->beFlushed = true; void* pBuff = NULL; int32_t code = streamStateSessionGet_rocksdb(getStateFileStore(pFileState), pKey, &pBuff, pVLen); if (code != TSDB_CODE_SUCCESS) { @@ -307,6 +309,7 @@ int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStream } pNewPos = getNewRowPosForWrite(pFileState); pNewPos->needFree = true; + pNewPos->beFlushed = true; } _end: @@ -482,6 +485,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void SRowBuffPos* pNewPos = getNewRowPosForWrite(pCur->pStreamFileState); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); pNewPos->needFree = true; + pNewPos->beFlushed = true; memcpy(pNewPos->pRowBuff, pData, *pVLen); (*pVal) = pNewPos; } diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 19b7359981..ea20f0e2b1 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -698,6 +698,7 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void stDebug("===stream===save skey:%" PRId64 ", ekey:%" PRId64 ", groupId:%" PRIu64 ".code:%d", key->win.skey, key->win.ekey, key->groupId, code); } else { + pos->beFlushed = false; code = putSessionWinResultBuff(pState->pFileState, value); } } diff --git a/tests/script/tsim/stream/partitionby1.sim b/tests/script/tsim/stream/partitionby1.sim index d92aecb3a6..24c588d410 100644 --- a/tests/script/tsim/stream/partitionby1.sim +++ b/tests/script/tsim/stream/partitionby1.sim @@ -13,6 +13,8 @@ sql create table ts3 using st tags(3,2,2); sql create table ts4 using st tags(4,2,2); sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST1 as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by tbname interval(10s); +sleep 1000 + sql insert into ts1 values(1648791213001,1,12,3,1.0); sql insert into ts2 values(1648791213001,1,12,3,1.0); From 2365054ada6aa591cf78f4cbae2ea63c280a0b85 Mon Sep 17 00:00:00 2001 From: zk66214 Date: Thu, 18 Jan 2024 20:09:03 +0800 Subject: [PATCH 38/81] add common function no_error to TDsql --- tests/pytest/util/sql.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 62af9a1af9..92074161b6 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -97,7 +97,24 @@ class TDSql: i+=1 time.sleep(1) pass - + + def no_error(self, sql): + caller = inspect.getframeinfo(inspect.stack()[1][0]) + expectErrOccurred = False + + try: + self.cursor.execute(sql) + except BaseException as e: + expectErrOccurred = True + self.errno = e.errno + error_info = repr(e) + self.error_info = ','.join(error_info[error_info.index('(') + 1:-1].split(",")[:-1]).replace("'", "") + + if expectErrOccurred: + tdLog.exit("%s(%d) failed: sql:%s, unexpect error '%s' occurred" % (caller.filename, caller.lineno, sql, self.error_info)) + else: + tdLog.info("sql:%s, check passed, no ErrInfo occurred" % (sql)) + def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True): caller = inspect.getframeinfo(inspect.stack()[1][0]) expectErrNotOccured = True @@ -126,9 +143,9 @@ class TDSql: if expectErrInfo != None: if expectErrInfo == self.error_info: - tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) + tdLog.info("sql:%s, expected ErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, ErrInfo '%s' occured, but not expected ErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) else: if expectedErrno != None: if expectedErrno in self.errno: @@ -138,9 +155,9 @@ class TDSql: if expectErrInfo != None: if expectErrInfo in self.error_info: - tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) + tdLog.info("sql:%s, expected ErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, ErrInfo %s occured, but not expected ErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) return self.error_info From ba48115231b9b5fb6413bc5e12f4adeb9053f465 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 19 Jan 2024 10:24:39 +0800 Subject: [PATCH 39/81] fix: heap user after free --- source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 1 + source/client/src/clientHb.c | 22 +++++++++++++--------- source/client/src/clientMsgHandler.c | 20 ++++++++++++++++++-- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index a6d5039be7..fd4776664c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -155,6 +155,7 @@ typedef struct STscObj { int8_t biMode; int32_t acctId; uint32_t connId; + int32_t appHbMgrIdx; int64_t id; // ref ID returned by taosAddRef TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b6c5701915..f1e9e36433 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -283,6 +283,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c pObj->connType = connType; pObj->pAppInfo = pAppInfo; + pObj->appHbMgrIdx = pAppInfo->pAppHbMgr->idx; tstrncpy(pObj->user, user, sizeof(pObj->user)); memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b3d9a9ced5..e076a874d3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -30,7 +30,7 @@ typedef struct { }; } SHbParam; -static SClientHbMgr clientHbMgr = {0}; +SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); @@ -1294,9 +1294,8 @@ void hbMgrCleanUp() { taosThreadMutexLock(&clientHbMgr.lock); appHbMgrCleanup(); - taosArrayDestroy(clientHbMgr.appHbMgrs); + clientHbMgr.appHbMgrs = taosArrayDestroy(clientHbMgr.appHbMgrs); taosThreadMutexUnlock(&clientHbMgr.lock); - clientHbMgr.appHbMgrs = NULL; } int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, int64_t clusterId) { @@ -1335,13 +1334,18 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) { - SAppHbMgr *pAppHbMgr = pTscObj->pAppInfo->pAppHbMgr; - SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - if (pReq) { - tFreeClientHbReq(pReq); - taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - taosHashRelease(pAppHbMgr->activeInfo, pReq); + SClientHbReq *pReq = NULL; + taosThreadMutexLock(&clientHbMgr.lock); + SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx); + if (pAppHbMgr) { + pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + if (pReq) { + tFreeClientHbReq(pReq); + taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + taosHashRelease(pAppHbMgr->activeInfo, pReq); + } } + taosThreadMutexUnlock(&clientHbMgr.lock); if (NULL == pReq) { return; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e0cedb9924..fbef9dfad4 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -26,6 +26,8 @@ #include "tname.h" #include "tversion.h" +extern SClientHbMgr clientHbMgr; + static void setErrno(SRequestObj* pRequest, int32_t code) { pRequest->code = code; terrno = code; @@ -63,12 +65,21 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { STscObj* pTscObj = pRequest->pTscObj; - if (NULL == pTscObj->pAppInfo || NULL == pTscObj->pAppInfo->pAppHbMgr) { + if (NULL == pTscObj->pAppInfo) { setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED); tsem_post(&pRequest->body.rspSem); goto End; } + taosThreadMutexLock(&clientHbMgr.lock); + if (NULL == taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx)) { + taosThreadMutexUnlock(&clientHbMgr.lock); + setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED); + tsem_post(&pRequest->body.rspSem); + goto End; + } + taosThreadMutexUnlock(&clientHbMgr.lock); + SConnectRsp connectRsp = {0}; if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) { code = TSDB_CODE_TSC_INVALID_VERSION; @@ -142,7 +153,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { pTscObj->authVer = connectRsp.authVer; pTscObj->whiteListInfo.ver = connectRsp.whiteListVer; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); + taosThreadMutexLock(&clientHbMgr.lock); + SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx); + if (pAppHbMgr) { + hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); + } + taosThreadMutexUnlock(&clientHbMgr.lock); tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, pTscObj->pAppInfo->numOfConns); From 1e3f70a1de2c56cf5b9cef0833734e374243de51 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 19 Jan 2024 14:04:22 +0800 Subject: [PATCH 40/81] test: add test of empty string in tmq --- tests/system-test/7-tmq/tmqCheckData1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index 1209c2812c..f443ce5b1e 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -51,7 +51,8 @@ class TDTestCase: tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) tdLog.info("insert data") tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) - + tdSql.execute("insert into ctb0 values(now,1,'');") + tdLog.info("create topics from stb with filter") queryString = "select ts,c1,c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"]) From ff06f9ef4260c92069fa993504da6af00b27bc78 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 19 Jan 2024 14:31:02 +0800 Subject: [PATCH 41/81] fix: heap user after free --- source/client/src/clientMsgHandler.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index fbef9dfad4..324b99022b 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -66,20 +66,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { STscObj* pTscObj = pRequest->pTscObj; if (NULL == pTscObj->pAppInfo) { - setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED); + code = TSDB_CODE_TSC_DISCONNECTED; + setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); goto End; } - taosThreadMutexLock(&clientHbMgr.lock); - if (NULL == taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx)) { - taosThreadMutexUnlock(&clientHbMgr.lock); - setErrno(pRequest, TSDB_CODE_TSC_DISCONNECTED); - tsem_post(&pRequest->body.rspSem); - goto End; - } - taosThreadMutexUnlock(&clientHbMgr.lock); - SConnectRsp connectRsp = {0}; if (tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp) != 0) { code = TSDB_CODE_TSC_INVALID_VERSION; @@ -106,7 +98,8 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { } if (connectRsp.epSet.numOfEps == 0) { - setErrno(pRequest, TSDB_CODE_APP_ERROR); + code = TSDB_CODE_APP_ERROR; + setErrno(pRequest, code); tsem_post(&pRequest->body.rspSem); goto End; } @@ -157,6 +150,12 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx); if (pAppHbMgr) { hbRegisterConn(pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); + } else { + taosThreadMutexUnlock(&clientHbMgr.lock); + code = TSDB_CODE_TSC_DISCONNECTED; + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + goto End; } taosThreadMutexUnlock(&clientHbMgr.lock); From bc9dfd8a8325bff3a7fc7d081991218a7d333eea Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 19 Jan 2024 14:42:02 +0800 Subject: [PATCH 42/81] fix: heap use after free --- source/client/src/clientHb.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e076a874d3..63a65d7c95 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1334,24 +1334,18 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } void hbDeregisterConn(STscObj *pTscObj, SClientHbKey connKey) { - SClientHbReq *pReq = NULL; taosThreadMutexLock(&clientHbMgr.lock); SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, pTscObj->appHbMgrIdx); if (pAppHbMgr) { - pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); taosHashRelease(pAppHbMgr->activeInfo, pReq); + atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } } taosThreadMutexUnlock(&clientHbMgr.lock); - - if (NULL == pReq) { - return; - } - - atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } // set heart beat thread quit mode , if quicByKill 1 then kill thread else quit from inner From f95d5e4f62037e3ea2489cf87103e13d2fb99d87 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 Jan 2024 15:28:23 +0800 Subject: [PATCH 43/81] opti:[TD-28118] raw block data for tmq --- include/libs/parser/parser.h | 2 +- source/client/src/clientRawBlockWrite.c | 3 +-- source/libs/parser/src/parInsertUtil.c | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 177607b178..bb206b5a02 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -150,7 +150,7 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int32_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); -int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); +int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 12ee67abbb..db8de44f1c 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1838,12 +1838,11 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } void* rawData = getRawDataFromRes(pRetrieve); - code = rawBlockBindData(pQuery, pTableMeta, rawData, pCreateReqDst, fields, pSW->nCols, true); + code = rawBlockBindData(pQuery, pTableMeta, rawData, &pCreateReqDst, fields, pSW->nCols, true); taosMemoryFree(fields); if (code != TSDB_CODE_SUCCESS) { goto end; } - pCreateReqDst = NULL; taosMemoryFreeClear(pTableMeta); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 9a9e73876d..6b389250bf 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -640,12 +640,12 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { return false; } -int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields, +int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields, int numFields, bool needChangeLength) { void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true, false); + sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; From 0e1ef540c4a633393ecd3ff1de6493a81c8ca4dc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 19 Jan 2024 17:09:02 +0800 Subject: [PATCH 44/81] opti:[TD-28118] raw block data for tmq --- source/libs/parser/src/parInsertUtil.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 6b389250bf..6b655bfae6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -247,13 +247,13 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat if (NULL == pTableCxt->pData) { code = TSDB_CODE_OUT_OF_MEMORY; } else { - pTableCxt->pData->flags = NULL != *pCreateTbReq ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; + pTableCxt->pData->flags = (pCreateTbReq != NULL && NULL != *pCreateTbReq) ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; pTableCxt->pData->suid = pTableMeta->suid; pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; - pTableCxt->pData->pCreateTbReq = *pCreateTbReq; - *pCreateTbReq = NULL; + pTableCxt->pData->pCreateTbReq = pCreateTbReq != NULL ? *pCreateTbReq : NULL; + if(pCreateTbReq != NULL) *pCreateTbReq = NULL; if (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { pTableCxt->pData->aCol = taosArrayInit(128, sizeof(SColData)); if (NULL == pTableCxt->pData->aCol) { From 547b75977e24a658bf52ae5aa02f968ee4e18aa8 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 19 Jan 2024 17:37:43 +0800 Subject: [PATCH 45/81] fix: timezone error on windows --- source/os/src/osTime.c | 70 ++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index e506ee603b..9cad1dd6ef 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -37,9 +37,6 @@ // This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) // until 00:00:00 January 1, 1970 static const uint64_t TIMEEPOCH = ((uint64_t)116444736000000000ULL); -// This magic number is the number of 100 nanosecond intervals since January 1, 1601 (UTC) -// until 00:00:00 January 1, 1900 -static const uint64_t TIMEEPOCH1900 = ((uint64_t)116445024000000000ULL); /* * We do not implement alternate representations. However, we always @@ -360,6 +357,7 @@ int32_t taosGetTimeOfDay(struct timeval *tv) { t.QuadPart -= TIMEEPOCH; tv->tv_sec = t.QuadPart / 10000000; tv->tv_usec = (t.QuadPart % 10000000) / 10; + return 0; #else return gettimeofday(tv, NULL); #endif @@ -482,33 +480,51 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result, char *buf) { sprintf(buf, "NaN"); } return NULL; - } + } else if (*timep < 0) { + SYSTEMTIME ss, s; + FILETIME ff, f; - SYSTEMTIME s; - FILETIME f; - LARGE_INTEGER offset; - struct tm tm1; - time_t tt = 0; - if (localtime_s(&tm1, &tt) != 0) { - if (buf != NULL) { - sprintf(buf, "NaN"); + LARGE_INTEGER offset; + struct tm tm1; + time_t tt = 0; + if (localtime_s(&tm1, &tt) != 0) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; + } + ss.wYear = tm1.tm_year + 1900; + ss.wMonth = tm1.tm_mon + 1; + ss.wDay = tm1.tm_mday; + ss.wHour = tm1.tm_hour; + ss.wMinute = tm1.tm_min; + ss.wSecond = tm1.tm_sec; + ss.wMilliseconds = 0; + SystemTimeToFileTime(&ss, &ff); + offset.QuadPart = ff.dwHighDateTime; + offset.QuadPart <<= 32; + offset.QuadPart |= ff.dwLowDateTime; + offset.QuadPart += *timep * 10000000; + f.dwLowDateTime = offset.QuadPart & 0xffffffff; + f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; + FileTimeToSystemTime(&f, &s); + result->tm_sec = s.wSecond; + result->tm_min = s.wMinute; + result->tm_hour = s.wHour; + result->tm_mday = s.wDay; + result->tm_mon = s.wMonth - 1; + result->tm_year = s.wYear - 1900; + result->tm_wday = s.wDayOfWeek; + result->tm_yday = 0; + result->tm_isdst = 0; + } else { + if (localtime_s(result, timep) != 0) { + if (buf != NULL) { + sprintf(buf, "NaN"); + } + return NULL; } - return NULL; } - offset.QuadPart = TIMEEPOCH1900; - offset.QuadPart += *timep * 10000000; - f.dwLowDateTime = offset.QuadPart & 0xffffffff; - f.dwHighDateTime = (offset.QuadPart >> 32) & 0xffffffff; - FileTimeToSystemTime(&f, &s); - result->tm_sec = s.wSecond; - result->tm_min = s.wMinute; - result->tm_hour = s.wHour; - result->tm_mday = s.wDay; - result->tm_mon = s.wMonth - 1; - result->tm_year = s.wYear - 1900; - result->tm_wday = s.wDayOfWeek; - result->tm_yday = 0; - result->tm_isdst = 0; #else res = localtime_r(timep, result); if (res == NULL && buf != NULL) { From 1e1951e86d0bebf4f6807b4e00c1e9126a8821f8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 19 Jan 2024 17:51:55 +0800 Subject: [PATCH 46/81] fix: add distributed total information --- tests/army/community/query/query_basic.py | 14 ++++++++++++ tests/army/frame/caseBase.py | 27 +++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 90916a1c6c..13b37ef253 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -76,12 +76,26 @@ class TDTestCase(TBase): sql2 = "select bi from stb where bi is not null order by bi desc limit 10;" self.checkSameResult(sql1, sql2) + # distributed expect values + expects = { + "Block_Rows" : 6*100000, + "Total_Tables" : 6, + "Total_Vgroups" : 3 + } + self.waitTransactionZero() + reals = self.getDistributed(self.stb) + for k in expects.keys(): + v = expects[k] + if int(reals[k]) != v: + tdLog.exit(f"distribute {k} expect: {v} real: {reals[k]}") + # run def run(self): tdLog.debug(f"start to excute {__file__}") # insert data self.insertData() + self.flushDb() # check insert data correct self.checkInsertCorrect() diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index b1500b69e1..5715867faf 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -229,9 +229,9 @@ class TBase: # # get vgroups - def getVGroup(self, db_name): + def getVGroup(self, dbName): vgidList = [] - sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{dbName}'" res = tdSql.getResult(sql) rows = len(res) for i in range(rows): @@ -239,6 +239,29 @@ class TBase: return vgidList + # get distributed rows + def getDistributed(self, tbName): + sql = f"show table distributed {tbName}" + tdSql.query(sql) + dics = {} + i = 0 + for i in range(tdSql.getRows()): + row = tdSql.getData(i, 0) + #print(row) + row = row.replace('[', '').replace(']', '') + #print(row) + items = row.split(' ') + #print(items) + for item in items: + #print(item) + v = item.split('=') + #print(v) + if len(v) == 2: + dics[v[0]] = v[1] + if i > 5: + break + print(dics) + return dics # From 897d3f5dba8209d80c4e297dc18cbb64c581b76f Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 19 Jan 2024 17:54:23 +0800 Subject: [PATCH 47/81] test:add special compatibility testcase for code coverage --- .../system-test/0-others/com_alltypedata.json | 81 ++++++ tests/system-test/0-others/compatibility.py | 7 + .../0-others/compatibility_coverage.py | 275 ++++++++++++++++++ 3 files changed, 363 insertions(+) create mode 100644 tests/system-test/0-others/com_alltypedata.json create mode 100644 tests/system-test/0-others/compatibility_coverage.py diff --git a/tests/system-test/0-others/com_alltypedata.json b/tests/system-test/0-others/com_alltypedata.json new file mode 100644 index 0000000000..0e6d8e3a07 --- /dev/null +++ b/tests/system-test/0-others/com_alltypedata.json @@ -0,0 +1,81 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 10, + "result_file": "./insert_res_mix.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "check_sql": "yes", + "continue_if_fail": "yes", + "databases": [ + { + "dbinfo": { + "name": "curdb", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "precision": "ms", + "stt_trigger": 8, + "minRows": 100, + "maxRows": 4096 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 5, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "insert_interval": 0, + "timestamp_step": 1000, + "start_timestamp":"2022-09-01 10:00:00", + "disorder_ratio": 60, + "update_ratio": 70, + "delete_ratio": 30, + "disorder_fill_interval": 300, + "update_fill_interval": 25, + "generate_row_rule": 2, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 1, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": 0 }, + { "type": "smallint", "name": "si", "max": 100, "min": 0 }, + { "type": "int", "name": "ic", "max": 100, "min": 0 }, + { "type": "bigint", "name": "bi", "max": 100, "min": 0 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 100, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 100, "min": 0 }, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + { + "type": "tinyint", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "binary", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index d54c676c0d..c936cf1ae4 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -152,6 +152,13 @@ class TDTestCase: tdLog.info(f" LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") os.system(f"LD_LIBRARY_PATH=/usr/lib taosBenchmark -t {tableNumbers} -n {recordNumbers1} -y ") os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database test '") + os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/com_alltypedata.json -y") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'flush database curdb '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select count(*) from curdb.meters '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select sum(fc) from curdb.meters '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select avg(ic) from curdb.meters '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select min(ui) from curdb.meters '") + os.system("LD_LIBRARY_PATH=/usr/lib taos -s 'select max(bi) from curdb.meters '") # os.system(f"LD_LIBRARY_PATH=/usr/lib taos -s 'use test;create stream current_stream into current_stream_output_stb as select _wstart as `start`, _wend as wend, max(current) as max_current from meters where voltage <= 220 interval (5s);' ") # os.system('LD_LIBRARY_PATH=/usr/lib taos -s "use test;create stream power_stream into power_stream_output_stb as select ts, concat_ws(\\".\\", location, tbname) as meter_location, current*voltage*cos(phase) as active_power, current*voltage*sin(phase) as reactive_power from meters partition by tbname;" ') diff --git a/tests/system-test/0-others/compatibility_coverage.py b/tests/system-test/0-others/compatibility_coverage.py new file mode 100644 index 0000000000..7a123739f7 --- /dev/null +++ b/tests/system-test/0-others/compatibility_coverage.py @@ -0,0 +1,275 @@ +from urllib.parse import uses_relative +import taos +import sys +import os +import time +import platform +import inspect +from taos.tmq import Consumer + +from pathlib import Path +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +import subprocess + +BASEVERSION = "3.0.2.3" +class TDTestCase: + def caseDescription(self): + f''' + 3.0 data compatibility test + case1: basedata version is {BASEVERSION} + ''' + return + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.deletedDataSql= '''drop database if exists deldata;create database deldata duration 300 stt_trigger 1; ;use deldata; + create table deldata.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int); + create table deldata.ct1 using deldata.stb1 tags ( 1 ); + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); + select avg(c1) from deldata.ct1; + delete from deldata.stb1; + flush database deldata; + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a ); + delete from deldata.ct1; + insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ); + flush database deldata;''' + def checkProcessPid(self,processName): + i=0 + while i<60: + print(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + print(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + sleep(1) + else: + print(f'this processName is not stoped in 60s') + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + self.projPath = projPath + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def getCfgPath(self): + buildPath = self.getBuildPath() + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + cfgPath = buildPath + "/../sim/dnode1/cfg/" + else: + cfgPath = buildPath + "/../sim/dnode1/cfg/" + + return cfgPath + + def installTaosd(self,bPath,cPath): + # os.system(f"rmtaos && mkdir -p {self.getBuildPath()}/build/lib/temp && mv {self.getBuildPath()}/build/lib/libtaos.so* {self.getBuildPath()}/build/lib/temp/ ") + # os.system(f" mv {bPath}/build {bPath}/build_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so {self.getBuildPath()}/build/lib/libtaos.so_bak ") + # os.system(f"mv {self.getBuildPath()}/build/lib/libtaos.so.1 {self.getBuildPath()}/build/lib/libtaos.so.1_bak ") + + packagePath = "/usr/local/src/" + dataPath = cPath + "/../data/" + if platform.system() == "Linux" and platform.machine() == "aarch64": + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-arm64.tar.gz" + else: + packageName = "TDengine-server-"+ BASEVERSION + "-Linux-x64.tar.gz" + packageTPath = packageName.split("-Linux-")[0] + my_file = Path(f"{packagePath}/{packageName}") + if not my_file.exists(): + print(f"{packageName} is not exists") + tdLog.info(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") + os.system(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") + else: + print(f"{packageName} has been exists") + os.system(f" cd {packagePath} && tar xvf {packageName} && cd {packageTPath} && ./install.sh -e no " ) + tdDnodes.stop(1) + print(f"start taosd: rm -rf {dataPath}/* && nohup taosd -c {cPath} & ") + os.system(f"rm -rf {dataPath}/* && nohup taosd -c {cPath} & " ) + sleep(5) + + + def buildTaosd(self,bPath): + # os.system(f"mv {bPath}/build_bak {bPath}/build ") + os.system(f" cd {bPath} ") + + def is_list_same_as_ordered_list(self,unordered_list, ordered_list): + sorted_list = sorted(unordered_list) + return sorted_list == ordered_list + + def run(self): + scriptsPath = os.path.dirname(os.path.realpath(__file__)) + distro_id = distro.id() + if distro_id == "alpine": + tdLog.info(f"alpine skip compatibility test") + return True + if platform.system().lower() == 'windows': + tdLog.info(f"Windows skip compatibility test") + return True + bPath = self.getBuildPath() + cPath = self.getCfgPath() + dbname = "test" + stb = f"{dbname}.meters" + os.system("echo 'debugFlag 143' > /etc/taos/taos.cfg ") + tableNumbers=100 + recordNumbers1=100 + recordNumbers2=1000 + + # tdsqlF=tdCom.newTdSql() + # print(tdsqlF) + # tdsqlF.query(f"SELECT SERVER_VERSION();") + # print(tdsqlF.query(f"SELECT SERVER_VERSION();")) + # oldServerVersion=tdsqlF.queryResult[0][0] + # tdLog.info(f"Base server version is {oldServerVersion}") + # tdsqlF.query(f"SELECT CLIENT_VERSION();") + # # the oldClientVersion can't be updated in the same python process,so the version is new compiled verison + # oldClientVersion=tdsqlF.queryResult[0][0] + # tdLog.info(f"Base client version is {oldClientVersion}") + # baseVersion = "3.0.1.8" + + tdLog.printNoPrefix(f"==========step1:prepare and check data in old version-{BASEVERSION}") + os.system(f"rm -rf {cPath}/../data") + print(self.projPath) + # this data file is special for coverage test in 192.168.1.96 + os.system("cp -r f{self.projPath}/../comp_testdata/data/ {self.projPath}/sim/dnode1") + tdDnodes.stop(1) + tdDnodes.start(1) + + + tdsql=tdCom.newTdSql() + tdsql.query(f"SELECT SERVER_VERSION();") + nowServerVersion=tdsql.queryResult[0][0] + tdLog.printNoPrefix(f"==========step3:prepare and check data in new version-{nowServerVersion}") + tdsql.query(f"select count(*) from {stb}") + tdsql.checkData(0,0,tableNumbers*recordNumbers1) + # tdsql.query("show streams;") + # os.system(f"taosBenchmark -t {tableNumbers} -n {recordNumbers2} -y ") + # tdsql.query("show streams;") + # tdsql.query(f"select count(*) from {stb}") + # tdsql.checkData(0,0,tableNumbers*recordNumbers2) + + # checkout db4096 + tdsql.query("select count(*) from db4096.stb0") + tdsql.checkData(0,0,50000) + + # checkout deleted data + tdsql.execute("insert into deldata.ct1 values ( now()-0s, 0, 0, 0, 0, 0.0, 0.0, 0, 'binary0', 'nchar0', now()+0a ) ( now()-10s, 1, 11111, 111, 11, 1.11, 11.11, 1, 'binary1', 'nchar1', now()+1a ) ( now()-20s, 2, 22222, 222, 22, 2.22, 22.22, 0, 'binary2', 'nchar2', now()+2a ) ( now()-30s, 3, 33333, 333, 33, 3.33, 33.33, 1, 'binary3', 'nchar3', now()+3a );") + tdsql.execute("flush database deldata;") + tdsql.query("select avg(c1) from deldata.ct1;") + + + tdsql=tdCom.newTdSql() + tdLog.printNoPrefix("==========step4:verify backticks in taos Sql-TD18542") + tdsql.execute("drop database if exists db") + tdsql.execute("create database db") + tdsql.execute("use db") + tdsql.execute("create stable db.stb1 (ts timestamp, c1 int) tags (t1 int);") + tdsql.execute("insert into db.ct1 using db.stb1 TAGS(1) values(now(),11);") + tdsql.error(" insert into `db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.error(" insert into db.`db.ct2` using db.stb1 TAGS(9) values(now(),11);") + tdsql.execute("insert into `db`.ct3 using db.stb1 TAGS(3) values(now(),13);") + tdsql.query("select * from db.ct3") + tdsql.checkData(0,1,13) + tdsql.execute("insert into db.`ct4` using db.stb1 TAGS(4) values(now(),14);") + tdsql.query("select * from db.ct4") + tdsql.checkData(0,1,14) + + #check retentions + tdsql=tdCom.newTdSql() + tdsql.query("describe information_schema.ins_databases;") + qRows=tdsql.queryRows + comFlag=True + j=0 + while comFlag: + for i in range(qRows) : + if tdsql.queryResult[i][0] == "retentions" : + print("parameters include retentions") + comFlag=False + break + else : + comFlag=True + j=j+1 + if j == qRows: + print("parameters don't include retentions") + caller = inspect.getframeinfo(inspect.stack()[0][0]) + args = (caller.filename, caller.lineno) + tdLog.exit("%s(%d) failed" % args) + + # check stream + tdsql.query("show streams;") + tdsql.checkRows(0) + + #check TS-3131 + tdsql.query("select *,tbname from d0.almlog where mcid='m0103';") + tdsql.checkRows(6) + expectList = [0,3003,20031,20032,20033,30031] + resultList = [] + for i in range(6): + resultList.append(tdsql.queryResult[i][3]) + print(resultList) + if self.is_list_same_as_ordered_list(resultList,expectList): + print("The unordered list is the same as the ordered list.") + else: + tdLog.exit("The unordered list is not the same as the ordered list.") + tdsql.execute("insert into test.d80 values (now+1s, 11, 103, 0.21);") + tdsql.execute("insert into test.d9 values (now+5s, 4.3, 104, 0.4);") + + + # check tmq + conn = taos.connect() + + consumer = Consumer( + { + "group.id": "tg75", + "client.id": "124", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "enable.auto.commit": "true", + "experimental.snapshot.enable": "true", + } + ) + consumer.subscribe(["tmq_test_topic"]) + + while True: + res = consumer.poll(10) + if not res: + break + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) + tdsql.query("show topics;") + tdsql.checkRows(1) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From e84d36dd392050873c2735bbf6fbfe9245b692b2 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 19 Jan 2024 22:42:10 +0800 Subject: [PATCH 48/81] fix: add test case --- source/libs/executor/test/timewindowTest.cpp | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index 3639bf15e1..047356b7a2 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -186,4 +186,45 @@ TEST(testCase, timewindow_natural) { ASSERT_EQ(w3.skey, w4.skey); ASSERT_EQ(w3.ekey, w4.ekey); } + +TEST(testCase, timewindow_natural) { + osSetTimezone("CST"); + + int32_t precision = TSDB_TIME_PRECISION_MILLI; + + SInterval interval2 = createInterval(17, 17, 13392000000, 'n', 'n', 0, precision); + int64_t key = 1648970865984; + STimeWindow w0 = getAlignQueryTimeWindow(&interval2, key); + printTimeWindow(&w0, precision, key); + ASSERT_GE(w0.ekey, key); + + int64_t key1 = 1633446027072; + STimeWindow w1 = {0}; + getInitialStartTimeWindow(&interval2, key1, &w1, true); + printTimeWindow(&w1, precision, key1); + STimeWindow w3 = getAlignQueryTimeWindow(&interval2, key1); + printf("%ld win %ld, %ld\n", key1, w3.skey, w3.ekey); + + int64_t key2 = 1648758398208; + STimeWindow w2 = {0}; + getInitialStartTimeWindow(&interval2, key2, &w2, true); + printTimeWindow(&w2, precision, key2); + STimeWindow w4 = getAlignQueryTimeWindow(&interval2, key2); + printf("%ld win %ld, %ld\n", key2, w3.skey, w3.ekey); + + ASSERT_EQ(w3.skey, w4.skey); + ASSERT_EQ(w3.ekey, w4.ekey); +} + +TEST(testCase, timewindow_active) { + osSetTimezone("CST"); + int32_t precision = TSDB_TIME_PRECISION_MILLI; + + SInterval interval = createInterval(10, 10, 2*365*24*60*60*1000, 'y', 'y', 0, precision); + SResultRowInfo dumyInfo = {0}; + dumyInfo.cur.pageId = -1; + int64_t key = 1609430400*1000; // 2021-01-01 + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, 1609430400*1000, &interval, TSDB_ORDER_ASC); + printTimeWindow(&win, precision, key); +} #pragma GCC diagnostic pop \ No newline at end of file From 95975801d52c1e4631b8a39b26285bdea3d148d1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 13:42:42 +0800 Subject: [PATCH 49/81] feat: add check max and ts window correctness --- tests/army/community/query/query_basic.json | 2 +- tests/army/community/query/query_basic.py | 142 +++++++++++++++++++- 2 files changed, 140 insertions(+), 4 deletions(-) diff --git a/tests/army/community/query/query_basic.json b/tests/army/community/query/query_basic.json index c6a3482cd4..d8d47266f9 100644 --- a/tests/army/community/query/query_basic.json +++ b/tests/army/community/query/query_basic.json @@ -32,7 +32,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 30000, - "start_timestamp":"2023-10-01 10:00:00", + "start_timestamp":1700000000000, "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 13b37ef253..eb332b8c71 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -34,23 +34,158 @@ class TDTestCase(TBase): "querySmaOptimize": "1" } + def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run jfile = etool.curFile(__file__, "query_basic.json") - etool.benchMark(json=jfile) + etool.benchMark(json = jfile) tdSql.execute(f"use {self.db}") tdSql.execute("select database();") - # set insert data information + # come from query_basic.json self.childtable_count = 6 self.insert_rows = 100000 self.timestamp_step = 30000 + self.start_timestamp = 1700000000000 + + + def genTime(self, preCnt, cnt): + start = self.start_timestamp + preCnt * self.timestamp_step + end = start + self.timestamp_step * cnt + return (start, end) + + + def doWindowQuery(self): + pre = f"select count(ts) from {self.stb} " + # case1 operator "in" "and" is same + cnt = 6000 + s,e = self.genTime(12000, cnt) + sql1 = f"{pre} where ts between {s} and {e} " + sql2 = f"{pre} where ts >= {s} and ts <={e} " + expectCnt = (cnt + 1) * self.childtable_count + tdSql.checkFirstValue(sql1, expectCnt) + tdSql.checkFirstValue(sql2, expectCnt) + + # case2 no overloap "or" left + cnt1 = 120 + s1, e1 = self.genTime(4000, cnt1) + cnt2 = 3000 + s2, e2 = self.genTime(10000, cnt2) + sql = f"{pre} where (ts >= {s1} and ts < {e1}) or (ts >= {s2} and ts < {e2})" + expectCnt = (cnt1 + cnt2) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case3 overloap "or" right + cnt1 = 300 + s1, e1 = self.genTime(17000, cnt1) + cnt2 = 8000 + s2, e2 = self.genTime(70000, cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})" + expectCnt = (cnt1 + cnt2) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case4 overloap "or" + cnt1 = 1000 + s1, e1 = self.genTime(9000, cnt1) + cnt2 = 1000 + s2, e2 = self.genTime(9000 + 500 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})" + expectCnt = (cnt1 + 500) * self.childtable_count # expect=1500 + tdSql.checkFirstValue(sql, expectCnt) + + # case5 overloap "or" boundary hollow->solid + cnt1 = 3000 + s1, e1 = self.genTime(45000, cnt1) + cnt2 = 2000 + s2, e2 = self.genTime(45000 + cnt1 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})" + expectCnt = (cnt1+cnt2) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case6 overloap "or" boundary solid->solid + cnt1 = 300 + s1, e1 = self.genTime(55000, cnt1) + cnt2 = 500 + s2, e2 = self.genTime(55000 + cnt1 , cnt2) + sql = f"{pre} where (ts >= {s1} and ts <= {e1}) or (ts >= {s2} and ts <= {e2})" + expectCnt = (cnt1+cnt2+1) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case7 overloap "and" + cnt1 = 1000 + s1, e1 = self.genTime(40000, cnt1) + cnt2 = 1000 + s2, e2 = self.genTime(40000 + 500 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts > {s2} and ts <= {e2})" + expectCnt = cnt1/2 * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case8 overloap "and" boundary hollow->solid solid->hollow + cnt1 = 3000 + s1, e1 = self.genTime(45000, cnt1) + cnt2 = 2000 + s2, e2 = self.genTime(45000 + cnt1 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts < {e2})" + expectCnt = 1 * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case9 no overloap "and" + cnt1 = 6000 + s1, e1 = self.genTime(20000, cnt1) + cnt2 = 300 + s2, e2 = self.genTime(70000, cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts <= {e2})" + expectCnt = 0 + tdSql.checkFirstValue(sql, expectCnt) + + # case10 cnt1 contain cnt2 and + cnt1 = 5000 + s1, e1 = self.genTime(25000, cnt1) + cnt2 = 400 + s2, e2 = self.genTime(28000, cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts < {e2})" + expectCnt = cnt2 * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + + def queryMax(self, colname): + sql = f"select max({colname}) from {self.stb}" + tdSql.query(sql) + return tdSql.getData(0, 0) + + + def checkMax(self): + # max for tsdbRetrieveDatablockSMA2 coverage + colname = "ui" + max = self.queryMax(colname) + + # insert over max + sql = f"insert into d0(ts, {colname}) values" + for i in range(1, 5): + sql += f" (now + {i}s, {max+i})" + tdSql.execute(sql) + self.flushDb() + + expectMax = max + 4 + for i in range(1, 5): + realMax = self.queryMax(colname) + if realMax != expectMax: + tdLog.exit(f"Max value not expect. expect:{expectMax} real:{realMax}") + sql = f"delete from d0 where ui={expectMax}" + tdSql.execute(sql) + expectMax -= 1 + + self.checkInsertCorrect() def doQuery(self): tdLog.info(f"do query.") - + self.doWindowQuery() + + # max + self.checkMax() + # __group_key sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti" tdSql.query(sql) @@ -89,6 +224,7 @@ class TDTestCase(TBase): if int(reals[k]) != v: tdLog.exit(f"distribute {k} expect: {v} real: {reals[k]}") + # run def run(self): tdLog.debug(f"start to excute {__file__}") From e1b0d4a555b383232241a39c83d728886d0ac60d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:26:34 +0800 Subject: [PATCH 50/81] fix: tweak delete max value --- tests/army/community/query/query_basic.py | 9 ++++++++- tests/army/frame/sql.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index eb332b8c71..575655c2b1 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -172,7 +172,14 @@ class TDTestCase(TBase): realMax = self.queryMax(colname) if realMax != expectMax: tdLog.exit(f"Max value not expect. expect:{expectMax} real:{realMax}") - sql = f"delete from d0 where ui={expectMax}" + + # query ts list + sql = f"select ts from d0 where ui={expectMax}" + tdSql.query(sql) + tss = tdSql.getColData(0) + strts = ",".join(tss) + # delete + sql = f"delete from d0 where ts in ({strts})" tdSql.execute(sql) expectMax -= 1 diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 8b7538321d..83f1243167 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -223,6 +223,12 @@ class TDSql: def getData(self, row, col): self.checkRowCol(row, col) return self.res[row][col] + + def getColData(self, col): + colDatas = [] + for i in range(self.queryRows): + colDatas.append(self.res[i][col]) + return colDatas def getResult(self, sql): self.sql = sql From 684bc221300e35cf352c8fbd9617d0ac9ccea75c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:26:41 +0800 Subject: [PATCH 51/81] fix: tweak delete max value --- tests/army/community/query/query_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 575655c2b1..740d14e11b 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -177,7 +177,7 @@ class TDTestCase(TBase): sql = f"select ts from d0 where ui={expectMax}" tdSql.query(sql) tss = tdSql.getColData(0) - strts = ",".join(tss) + strts = ",".join(map(str,tss)) # delete sql = f"delete from d0 where ts in ({strts})" tdSql.execute(sql) From 6cd689ea53714bf8035eb32bb66f61badd6c428c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:43:00 +0800 Subject: [PATCH 52/81] fix: delete max ok --- tests/army/community/query/query_basic.py | 8 ++++---- tests/army/frame/caseBase.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 740d14e11b..211d3e6ec3 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -177,10 +177,10 @@ class TDTestCase(TBase): sql = f"select ts from d0 where ui={expectMax}" tdSql.query(sql) tss = tdSql.getColData(0) - strts = ",".join(map(str,tss)) - # delete - sql = f"delete from d0 where ts in ({strts})" - tdSql.execute(sql) + for ts in tss: + # delete + sql = f"delete from d0 where ts = '{ts}'" + tdSql.execute(sql) expectMax -= 1 self.checkInsertCorrect() diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 5715867faf..de258e6267 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -292,3 +292,15 @@ class TBase: if len(lists) == 0: tdLog.exit(f"list is empty {tips}") + +# +# str util +# + # covert list to sql format string + def listSql(self, lists, sepa = ","): + strs = "" + for ls in lists: + if strs != "": + strs += sepa + strs += f"'{ls}'" + return strs \ No newline at end of file From 68b2a06a02289bea1d9e4e8f4d93530eac3627a0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:52:40 +0800 Subject: [PATCH 53/81] feat: add cquery_basic.json write again --- tests/army/community/query/cquery_basic.json | 61 ++++++++++++++++++++ tests/army/community/query/query_basic.py | 6 +- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/army/community/query/cquery_basic.json diff --git a/tests/army/community/query/cquery_basic.json b/tests/army/community/query/cquery_basic.json new file mode 100644 index 0000000000..5a57d59d93 --- /dev/null +++ b/tests/army/community/query/cquery_basic.json @@ -0,0 +1,61 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 4000, + "prepared_rand": 10000, + "thread_count": 3, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "no", + "vgroups": 3, + "replica": 3, + "duration":"3d", + "wal_retention_period": 1, + "wal_retention_size": 1, + "stt_trigger": 1 + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "yes", + "childtable_count": 6, + "insert_rows": 50000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 60000, + "start_timestamp":1700000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 8}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 211d3e6ec3..912974d8ab 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -49,6 +49,11 @@ class TDTestCase(TBase): self.timestamp_step = 30000 self.start_timestamp = 1700000000000 + # write again disorder + self.flushDb() + jfile = etool.curFile(__file__, "cquery_basic.json") + etool.benchMark(json = jfile) + def genTime(self, preCnt, cnt): start = self.start_timestamp + preCnt * self.timestamp_step @@ -238,7 +243,6 @@ class TDTestCase(TBase): # insert data self.insertData() - self.flushDb() # check insert data correct self.checkInsertCorrect() From c2772994e43d7fa08a54c95f6ece2ee5040636e4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 15:00:14 +0800 Subject: [PATCH 54/81] fix: add order by desc coverage --- tests/army/frame/caseBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index de258e6267..f562ea1086 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -136,7 +136,7 @@ class TBase: tdSql.checkAgg(sql, self.childtable_count) # check step - sql = f"select * from (select diff(ts) as dif from {self.stb} partition by tbname) where dif != {self.timestamp_step}" + sql = f"select * from (select diff(ts) as dif from {self.stb} partition by tbname order by ts desc) where dif != {self.timestamp_step}" tdSql.query(sql) tdSql.checkRows(0) From 4c804904da6ea50f123f13fc22887f957adf7538 Mon Sep 17 00:00:00 2001 From: menshibin Date: Sat, 20 Jan 2024 16:19:14 +0800 Subject: [PATCH 55/81] Resolve conflicts --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1c2f5ec23c..766afc5358 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,6 +11,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py From db9b004c5816c63723c0e0cb58894ba75879d003 Mon Sep 17 00:00:00 2001 From: menshibin Date: Sat, 20 Jan 2024 16:36:34 +0800 Subject: [PATCH 56/81] Annotations in English --- tests/army/frame/sql.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index b5b5f836b9..d2a6292f8e 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -467,17 +467,15 @@ class TDSql: data = [] try: with open(csvfilePath) as csvfile: - csv_reader = csv.reader(csvfile) # 使用csv.reader读取csvfile中的文件 - # header = next(csv_reader) # 读取第一行每一列的标题 - for row in csv_reader: # 将csv 文件中的数据保存到data中 - data.append(row) # 选择某一列加入到data数组中 + csv_reader = csv.reader(csvfile) # csv.reader read csvfile\ + # header = next(csv_reader) # Read the header of each column in the first row + for row in csv_reader: # csv file save to data + data.append(row) except FileNotFoundError: - # 当文件不存在时会引发FileNotFoundError异常 caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, csvfilePath) tdLog.exit("%s(%d) failed: sql:%s, expect csvfile not find error:%s" % args) except Exception as e: - # 其他任意类型的异常都将被捕获并输出相应信息 caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, self.sql, csvfilePath, str(e)) tdLog.exit("%s(%d) failed: sql:%s, expect csvfile path:%s, read error:%s" % args) From 09c372a456907dc907de0495e5b7220da8ea38f1 Mon Sep 17 00:00:00 2001 From: menshibin Date: Sat, 20 Jan 2024 16:57:24 +0800 Subject: [PATCH 57/81] Resolve conflicts --- tests/army/community/query/fill/fill_desc.py | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/army/community/query/fill/fill_desc.py b/tests/army/community/query/fill/fill_desc.py index 61638e73ed..170c34ec49 100644 --- a/tests/army/community/query/fill/fill_desc.py +++ b/tests/army/community/query/fill/fill_desc.py @@ -34,12 +34,21 @@ class TDTestCase(TBase): f'''create table if not exists {dbname}.{tbname} using {dbname}.{stbname} tags("California.SanFrancisco", 2)''' ) - tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:35:00.000', 5.0)") - tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:36:00.000', 5.0)") - tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:37:00.000', 5.0)") - tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:38:00.000', null)") - tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:39:00.000', 5.0)") - tdSql.execute(f"insert into {dbname}.{tbname} values('2023-12-26 10:40:00.000', null)") + sqls = [] + for i in range(35, 41): + if i == 38 or i == 40: + sqls.append(f"insert into {dbname}.{tbname} values('2023-12-26 10:{i}:00.000', null)") + else: + sqls.append(f"insert into {dbname}.{tbname} values('2023-12-26 10:{i}:00.000', 5.0)") + + # sqls.append(f"insert into {dbname}.{tbname} values('2023-12-26 10:36:00.000', 5.0)") + # sqls.append(f"insert into {dbname}.{tbname} values('2023-12-26 10:37:00.000', 5.0)") + # sqls.append(f"insert into {dbname}.{tbname} values('2023-12-26 10:38:00.000', null)") + # sqls.append(f"insert into {dbname}.{tbname} values('2023-12-26 10:39:00.000', 5.0)") + # sqls.append(f"insert into {dbname}.{tbname} values('2023-12-26 10:40:00.000', null)") + + + tdSql.executes(sqls) tdLog.printNoPrefix("==========step3:fill data") From 9e3a42af01806735236f8b4fd7d79c6f6c43d916 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sun, 21 Jan 2024 16:15:51 +0800 Subject: [PATCH 58/81] fix: modify test case --- source/libs/executor/test/timewindowTest.cpp | 40 ++++---------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/source/libs/executor/test/timewindowTest.cpp b/source/libs/executor/test/timewindowTest.cpp index 047356b7a2..7ccbf0b10f 100644 --- a/source/libs/executor/test/timewindowTest.cpp +++ b/source/libs/executor/test/timewindowTest.cpp @@ -19,6 +19,7 @@ #include "thash.h" #include "tsimplehash.h" #include "executor.h" +#include "executorInt.h" #include "ttime.h" #pragma GCC diagnostic push @@ -187,44 +188,19 @@ TEST(testCase, timewindow_natural) { ASSERT_EQ(w3.ekey, w4.ekey); } -TEST(testCase, timewindow_natural) { - osSetTimezone("CST"); - - int32_t precision = TSDB_TIME_PRECISION_MILLI; - - SInterval interval2 = createInterval(17, 17, 13392000000, 'n', 'n', 0, precision); - int64_t key = 1648970865984; - STimeWindow w0 = getAlignQueryTimeWindow(&interval2, key); - printTimeWindow(&w0, precision, key); - ASSERT_GE(w0.ekey, key); - - int64_t key1 = 1633446027072; - STimeWindow w1 = {0}; - getInitialStartTimeWindow(&interval2, key1, &w1, true); - printTimeWindow(&w1, precision, key1); - STimeWindow w3 = getAlignQueryTimeWindow(&interval2, key1); - printf("%ld win %ld, %ld\n", key1, w3.skey, w3.ekey); - - int64_t key2 = 1648758398208; - STimeWindow w2 = {0}; - getInitialStartTimeWindow(&interval2, key2, &w2, true); - printTimeWindow(&w2, precision, key2); - STimeWindow w4 = getAlignQueryTimeWindow(&interval2, key2); - printf("%ld win %ld, %ld\n", key2, w3.skey, w3.ekey); - - ASSERT_EQ(w3.skey, w4.skey); - ASSERT_EQ(w3.ekey, w4.ekey); -} TEST(testCase, timewindow_active) { osSetTimezone("CST"); int32_t precision = TSDB_TIME_PRECISION_MILLI; - - SInterval interval = createInterval(10, 10, 2*365*24*60*60*1000, 'y', 'y', 0, precision); + int64_t offset = (int64_t)2*365*24*60*60*1000; + SInterval interval = createInterval(10, 10, offset, 'y', 'y', 0, precision); SResultRowInfo dumyInfo = {0}; dumyInfo.cur.pageId = -1; - int64_t key = 1609430400*1000; // 2021-01-01 - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, 1609430400*1000, &interval, TSDB_ORDER_ASC); + int64_t key = (int64_t)1609430400*1000; // 2021-01-01 + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, key, &interval, TSDB_ORDER_ASC); printTimeWindow(&win, precision, key); + printf("%ld win %ld, %ld\n", key, win.skey, win.ekey); + ASSERT_EQ(win.skey, 1325376000000); + ASSERT_EQ(win.ekey, 1640908799999); } #pragma GCC diagnostic pop \ No newline at end of file From 53b385108c1f0478adc3d18b7536a1980a56091e Mon Sep 17 00:00:00 2001 From: menshibin Date: Sun, 21 Jan 2024 18:38:25 +0800 Subject: [PATCH 59/81] add use db --- tests/army/community/cluster/incSnapshot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 9f50ea52f1..b1b53e65bd 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -29,6 +29,7 @@ class TDTestCase(TBase): tdSql.prepare() autoGen = AutoGen() autoGen.create_db(self.db, 2, 3) + tdSql.execute(f"use {self.db}") autoGen.create_stable(self.stb, 5, 10, 8, 8) autoGen.create_child(self.stb, "d", self.childtable_count) autoGen.insert_data(1000) From 091da20ab827e4d1ab4b5733eb7dad6fb24ce8db Mon Sep 17 00:00:00 2001 From: menshibin Date: Sun, 21 Jan 2024 20:40:06 +0800 Subject: [PATCH 60/81] add use db --- tests/army/frame/autogen.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index d38d2e2f19..bbcbf9ad9a 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -163,22 +163,18 @@ class AutoGen: values = "" tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") - return ts + self.ts = ts def insert_data(self, cnt, bContinue=False): if not bContinue: self.ts = 1600000000000 - currTs = 1600000000000 for i in range(self.child_cnt): name = f"{self.child_name}{i}" - currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + self.insert_data_child(name, cnt, self.batch_size, 1) - self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") - # insert same timestamp to all childs - # insert same timestamp to all childs def insert_samets(self, cnt): for i in range(self.child_cnt): From 9a8660dd4e30f49e531b9a73210911911da53ad6 Mon Sep 17 00:00:00 2001 From: menshibin Date: Sun, 21 Jan 2024 20:43:07 +0800 Subject: [PATCH 61/81] modify insert ts contiune --- tests/army/frame/autogen.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index bbcbf9ad9a..d1f02e7865 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -163,16 +163,18 @@ class AutoGen: values = "" tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") - self.ts = ts + return ts def insert_data(self, cnt, bContinue=False): if not bContinue: self.ts = 1600000000000 + currTs = 1600000000000 for i in range(self.child_cnt): name = f"{self.child_name}{i}" - self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") # insert same timestamp to all childs From a7713666207a4a9ce3be25e7464f1f2a788c5bdc Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 22 Jan 2024 09:33:34 +0800 Subject: [PATCH 62/81] test: modify testcase of tmqvnodesplit --- tests/parallel_test/cases.task | 6 +- .../7-tmq/tmqVnodeSplit-column-false.py | 217 +++++++++++++++++ .../system-test/7-tmq/tmqVnodeSplit-column.py | 2 - .../7-tmq/tmqVnodeSplit-db-false.py | 218 ++++++++++++++++++ tests/system-test/7-tmq/tmqVnodeSplit-db.py | 2 - 5 files changed, 440 insertions(+), 5 deletions(-) create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-column-false.py create mode 100644 tests/system-test/7-tmq/tmqVnodeSplit-db-false.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0687a3271c..895b7d7697 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -231,10 +231,14 @@ fi ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-select-false.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-stb-false.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column.py -N 3 -n 3 +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-column-false.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db.py -N 3 -n 3 -e +,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeSplit-db-false.py -N 3 -n 3 + ,,y,system-test,./pytest.sh python3 test.py -f 7-tmq/tmqVnodeReplicate.py -M 3 -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py new file mode 100644 index 0000000000..9c3179f6a8 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py @@ -0,0 +1,217 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s where c2 >= 0 "%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 > resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column.py b/tests/system-test/7-tmq/tmqVnodeSplit-column.py index 74213c4536..8987cf5251 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column.py @@ -206,8 +206,6 @@ class TDTestCase: def run(self): self.prepareTestEnv() self.tmqCase1(True) - self.prepareTestEnv() - self.tmqCase1(False) def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py new file mode 100644 index 0000000000..d47874dc39 --- /dev/null +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py @@ -0,0 +1,218 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * +sys.path.append("./7-tmq") +from tmqCommon import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +class TDTestCase: + def __init__(self): + self.vgroups = 1 + self.ctbNum = 10 + self.rowsPerTbl = 1000 + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def getDataPath(self): + selfPath = tdCom.getBuildPath() + + return selfPath + '/../sim/dnode%d/data/vnode/vnode%d/wal/*'; + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdCom.drop_all_db() + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], wal_retention_period=36000,vgroups=paraDict["vgroups"],replica=self.replicaVar) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + return + + def restartAndRemoveWal(self, deleteWal): + tdDnodes = cluster.dnodes + tdSql.query("select * from information_schema.ins_vnodes") + for result in tdSql.queryResult: + if result[2] == 'dbt': + tdLog.debug("dnode is %d"%(result[0])) + dnodeId = result[0] + vnodeId = result[1] + + tdDnodes[dnodeId - 1].stoptaosd() + time.sleep(1) + dataPath = self.getDataPath() + dataPath = dataPath%(dnodeId,vnodeId) + tdLog.debug("dataPath:%s"%dataPath) + if deleteWal: + if os.system('rm -rf ' + dataPath) != 0: + tdLog.exit("rm error") + + tdDnodes[dnodeId - 1].starttaosd() + time.sleep(1) + break + tdLog.debug("restart dnode ok") + + def splitVgroups(self): + tdSql.query("select * from information_schema.ins_vnodes") + vnodeId = 0 + for result in tdSql.queryResult: + if result[2] == 'dbt': + vnodeId = result[1] + tdLog.debug("vnode is %d"%(vnodeId)) + break + splitSql = "split vgroup %d" %(vnodeId) + tdLog.debug("splitSql:%s"%(splitSql)) + tdSql.query(splitSql) + tdLog.debug("splitSql ok") + + def tmqCase1(self, deleteWal=False): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb1', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 60, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + # expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from db") + queryString = "database %s"%(paraDict['dbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # tdSql.query(queryString) + # expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:200, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + tdLog.info("create ctb1") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("create ctb2") + paraDict2 = paraDict.copy() + + paraDict2['ctbPrefix'] = "ctb2" + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict2['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("insert ctb1 data") + pInsertThread = tmqCom.asyncInsertDataByInterlace(paraDict) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() + + #restart dnode & remove wal + self.restartAndRemoveWal(deleteWal) + + # split vgroup + self.splitVgroups() + + + tdLog.info("insert ctb2 data") + pInsertThread1 = tmqCom.asyncInsertDataByInterlace(paraDict2) + pInsertThread.join() + pInsertThread1.join() + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectrowcnt / 2 > resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(2) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + if deleteWal == True: + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + self.prepareTestEnv() + self.tmqCase1(False) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db.py b/tests/system-test/7-tmq/tmqVnodeSplit-db.py index b9eac4b692..a9fb1c2d4b 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db.py @@ -207,8 +207,6 @@ class TDTestCase: def run(self): self.prepareTestEnv() self.tmqCase1(True) - self.prepareTestEnv() - self.tmqCase1(False) def stop(self): tdSql.close() From d593f367f0975857c35ae025139b7e970301a01a Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 22 Jan 2024 09:40:23 +0800 Subject: [PATCH 63/81] test: increase timeout in case of tmqvnodesplit --- tests/system-test/7-tmq/tmqVnodeSplit-column-false.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-db-false.py | 2 +- tests/system-test/7-tmq/tmqVnodeSplit-stb-false.py | 2 +- .../7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py index 9c3179f6a8..6ef28a4e77 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-column-false.py @@ -121,7 +121,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 180, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py index d47874dc39..bad9e09da5 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-db-false.py @@ -52,7 +52,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 180, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-false.py index 384959c52b..1e8b90d11e 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-false.py @@ -54,7 +54,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 60, + 'pollDelay': 180, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py index f6b523e5f4..3965168fa7 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-duplicatedata-false.py @@ -123,7 +123,7 @@ class TDTestCase: 'rowsPerTbl': 1000, 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 120, + 'pollDelay': 180, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} From c3c8d7b07d6f57742b9123fe9d4d8288b788f5be Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 22 Jan 2024 10:11:06 +0800 Subject: [PATCH 64/81] fix: compute scalar functions before agg in session window --- source/libs/executor/src/timewindowoperator.c | 15 +++++++++++++++ source/libs/planner/src/planSpliter.c | 5 ++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b86253a8d1..59455840b0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -30,6 +30,7 @@ typedef struct SSessionAggOperatorInfo { SOptrBasicInfo binfo; SAggSupporter aggSup; + SExprSupp scalarSupp; // supporter for perform scalar function SGroupResInfo groupResInfo; SWindowRowsSup winSup; bool reptScan; // next round scan @@ -1407,6 +1408,10 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { } pBInfo->pRes->info.scanFlag = pBlock->info.scanFlag; + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true); blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId); @@ -1570,6 +1575,16 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW pInfo->reptScan = false; pInfo->binfo.inputTsOrder = pSessionNode->window.node.inputTsOrder; pInfo->binfo.outputTsOrder = pSessionNode->window.node.outputTsOrder; + + if (pSessionNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + code = filterInitFromNode((SNode*)pSessionNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { goto _error; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index f2258a1d7e..28e31b7a4f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1064,9 +1064,8 @@ static int32_t stbSplCreateMergeKeys(SNodeList* pSortKeys, SNodeList* pTargets, SNode* pTarget = NULL; bool found = false; FOREACH(pTarget, pTargets) { - if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) - // || (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName)) - ) { + if ((QUERY_NODE_COLUMN == nodeType(pSortExpr) && nodesEqualNode((SNode*)pSortExpr, pTarget)) || + (0 == strcmp(pSortExpr->aliasName, ((SColumnNode*)pTarget)->colName))) { code = nodesListMakeStrictAppend(&pMergeKeys, stbSplCreateOrderByExpr(pSortKey, pTarget)); if (TSDB_CODE_SUCCESS != code) { break; From 0e525a8e224e86e6d4bf9fa92c6b651aefefc445 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 18 Jan 2024 14:57:12 +0800 Subject: [PATCH 65/81] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/partition_expr.py | 35 +++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/system-test/2-query/partition_expr.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0687a3271c..8eeecce750 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -43,6 +43,7 @@ fi #,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 ,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_expr.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/project_group.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_interval.py diff --git a/tests/system-test/2-query/partition_expr.py b/tests/system-test/2-query/partition_expr.py new file mode 100644 index 0000000000..c03d7eccb3 --- /dev/null +++ b/tests/system-test/2-query/partition_expr.py @@ -0,0 +1,35 @@ +from wsgiref.headers import tspecials +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.batchNum = 5 + self.ts = 1537146000000 + + def run(self): + dbname = "db" + tdSql.prepare() + + tdSql.execute(f'''create table sta(ts timestamp, f int, col2 bigint) tags(tg1 int, tg2 binary(20))''') + tdSql.execute(f"create table sta1 using sta tags(1, 'a')") + tdSql.execute(f"insert into sta1 values(1537146000001, 11, 110)") + tdSql.execute(f"insert into sta1 values(1537146000002, 12, 120)") + tdSql.execute(f"insert into sta1 values(1537146000003, 13, 130)") + + tdSql.query("select _wstart, f+100, count(*) from db.sta partition by f+100 session(ts, 1a) order by _wstart"); + tdSql.checkData(0, 1, 111.0) + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 00fd4f21cad0d58c9ea7c08e543be6f42c43de69 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 22 Jan 2024 10:14:11 +0800 Subject: [PATCH 66/81] fix: order by same name col --- source/libs/parser/src/parTranslater.c | 3 +++ tests/system-test/2-query/orderBy.py | 35 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 70e4744644..a2eeac6781 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4515,6 +4515,9 @@ static EDealRes replaceOrderByAliasImpl(SNode** pNode, void* pContext) { FOREACH(pProject, pProjectionList) { SExprNode* pExpr = (SExprNode*)pProject; if (0 == strcmp(((SColumnRefNode*)*pNode)->colName, pExpr->userAlias) && nodeType(*pNode) == nodeType(pProject)) { + if (QUERY_NODE_COLUMN == nodeType(pProject) && !nodesEqualNode(*pNode, pProject)) { + continue; + } SNode* pNew = nodesCloneNode(pProject); if (NULL == pNew) { pCxt->pTranslateCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; diff --git a/tests/system-test/2-query/orderBy.py b/tests/system-test/2-query/orderBy.py index 1cf1478439..2d7b7d9a1f 100644 --- a/tests/system-test/2-query/orderBy.py +++ b/tests/system-test/2-query/orderBy.py @@ -315,6 +315,39 @@ class TDTestCase: tdSql.no_error('select c1 as name from (select c1, c2 as name from st) order by name') + def queryOrderBySameCol(self): + tdLog.info("query OrderBy same col ....") + tdSql.execute(f"create stable sta (ts timestamp, col1 int) tags(t1 int);") + tdSql.execute(f"create table tba1 using sta tags(1);") + tdSql.execute(f"create table tba2 using sta tags(2);") + + pd = datetime.datetime.now() + ts = int(datetime.datetime.timestamp(pd)*1000*1000) + tdSql.execute(f"insert into tba1 values ({ts}, 1);") + tdSql.execute(f"insert into tba1 values ({ts+2}, 3);") + tdSql.execute(f"insert into tba1 values ({ts+3}, 4);") + tdSql.execute(f"insert into tba1 values ({ts+4}, 5);") + tdSql.execute(f"insert into tba2 values ({ts}, 2);") + tdSql.execute(f"insert into tba2 values ({ts+1}, 3);") + tdSql.execute(f"insert into tba2 values ({ts+3}, 5);") + tdSql.execute(f"insert into tba2 values ({ts+5}, 7);") + tdSql.query(f"select a.col1, b.col1 from sta a inner join sta b on a.ts = b.ts and a.ts < {ts+2} order by a.col1, b.col1;") + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 1, 2) + tdSql.query(f"select a.col1, b.col1 from sta a inner join sta b on a.ts = b.ts and a.ts < {ts+2} order by a.col1, b.col1 desc;") + tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 1, 2) + tdSql.checkData(1, 0, 1) + tdSql.checkData(1, 1, 1) + + tdSql.query(f"select a.col1, b.col1 from sta a inner join sta b on a.ts = b.ts and a.ts < {ts+2} order by a.col1 desc, b.col1 desc;") + tdSql.checkData(1, 0, 2) + tdSql.checkData(1, 1, 2) + tdSql.checkData(2, 0, 2) + tdSql.checkData(2, 1, 1) + # run def run(self): # prepare env @@ -332,6 +365,8 @@ class TDTestCase: # td-28332 self.queryOrderByAmbiguousName() + self.queryOrderBySameCol() + # stop def stop(self): tdSql.close() From 31f0fe8c4f20eaae95370ecb492dca8685821604 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 22 Jan 2024 10:23:53 +0800 Subject: [PATCH 67/81] docs:opti format of schemaless & modify subtable rules in stream --- docs/en/12-taos-sql/14-stream.md | 2 +- .../13-schemaless/13-schemaless.md | 24 ++++++++++-------- docs/zh/12-taos-sql/14-stream.md | 8 +++--- .../13-schemaless/13-schemaless.md | 25 ++++++++++--------- 4 files changed, 32 insertions(+), 27 deletions(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index c8be1753a2..6f2343d347 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -67,7 +67,7 @@ If a stream is created with PARTITION BY clause and SUBTABLE clause, the name of CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m); ``` -IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name. Other expressions are also allowed in SUBTABLE clause, but the output type must be varchar. +IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name(Starting from 3.2.3.0, in order to avoid the expression in subtable being unable to distinguish between different subtables, add '_groupId' to the end of subtable name). If the output length exceeds the limitation of TDengine(192), the name will be truncated. If the generated name is occupied by some other table, the creation and writing of the new subtable will be failed. diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 9b001ee79c..4a259f100c 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -84,18 +84,22 @@ Schemaless writes process row data according to the following principles. 1. You can use the following rules to generate the subtable names: first, combine the measurement name and the key and value of the label into the next string: -```json -"measurement,tag_key1=tag_value1,tag_key2=tag_value2" -``` + ```json + "measurement,tag_key1=tag_value1,tag_key2=tag_value2" + ``` -:::tip -Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol. -The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t\_" is a fixed prefix that every table generated by this mapping relationship has. -::: + :::tip + Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol. + The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t\_" is a fixed prefix that every table generated by this mapping relationship has. + ::: -If you do not want to use an automatically generated table name, there are two ways to specify sub table names, the first one has a higher priority. -You can configure smlAutoChildTableNameDelimiter in taos.cfg(except for `@ # space \r \t \n`), for example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. -You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. + If you do not want to use an automatically generated table name, there are two ways to specify sub table names(the first one has a higher priority). + + 1. You can configure smlAutoChildTableNameDelimiter in taos.cfg(except for `@ # space \r \t \n`). + 1. For example, `smlAutoChildTableNameDelimiter=tname`. You can insert `st,t0=cpul,t1=4 c1=3 1626006833639000000` and the table name will be cpu1-4. + + 2. You can configure smlChildTableName in taos.cfg to specify table names. + 2. For example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. **Important:** Manually creating supertables for schemaless writing is not supported. Schemaless writing creates appropriate supertables automatically. diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 929cf9ee4e..979fc436b9 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -34,7 +34,7 @@ subquery: SELECT select_list stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 写入已存在的超级表 -TAGS 字句定义了流计算中创建TAG的规则，可以为每个partition对应的子表生成自定义的TAG值，详见 自定义TAG +TAGS 子句定义了流计算中创建TAG的规则，可以为每个partition对应的子表生成自定义的TAG值，详见 自定义TAG ```sql create_definition: col_name column_definition @@ -77,7 +77,7 @@ SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL( CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m); ``` -PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名。 +PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名(从3.2.3.0开始，为了避免 sutable 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 _groupId)。 注意，子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。 @@ -212,8 +212,8 @@ CREATE STREAM streams2 trigger at_once INTO st1 TAGS(cc varchar(100)) as select PARTITION 子句中，为 concat("tag-", tbname)定义了一个别名cc, 对应超级表st1的自定义TAG的名字。在上述示例中，流新创建的子表的TAG将以前缀 'new-' 连接原表名作为TAG的值。 会对TAG信息进行如下检查 -1.检查tag的schema信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。 -2.检查tag的个数是否相同，如果不同，需要显示的指定超级表与subquery的tag的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 +1. 检查tag的schema信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。 +2. 检查tag的个数是否相同，如果不同，需要显示的指定超级表与subquery的tag的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 ## 清理中间状态 diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 723531676c..47737ab7ee 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -87,19 +87,20 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 1. 将使用如下规则来生成子表名：首先将 measurement 的名称和标签的 key 和 value 组合成为如下的字符串 -```json -"measurement,tag_key1=tag_value1,tag_key2=tag_value2" -``` + ```json + "measurement,tag_key1=tag_value1,tag_key2=tag_value2" + ``` -:::tip -需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。 -排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 -:::tip -如果不想用自动生成的表名，有两种指定子表名的方式，第一种优先级更高： -通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定（`@ # 空格 回车 换行 制表符`除外)。 -举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 -通过在taos.cfg里配置 smlChildTableName 参数来指定。 -举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 + :::tip + 需要注意的是，这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序，而是使用了标签名称按照字符串升序排列后的结果。所以，tag_key1 并不是在行协议中输入的第一个标签。 + 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名：“t_md5_val”。其中的 “t_” 是固定的前缀，每个通过该映射关系自动生成的表都具有该前缀。 + :::tip + + 如果不想用自动生成的表名，有两种指定子表名的方式(第一种优先级更高)。 + 1. 通过在taos.cfg里配置 smlAutoChildTableNameDelimiter 参数来指定（`@ # 空格 回车 换行 制表符`除外)。 + 1. 举例如下：配置 smlAutoChildTableNameDelimiter=- 插入数据为 st,t0=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1-4。 + 2. 通过在taos.cfg里配置 smlChildTableName 参数来指定。 + 1. 举例如下：配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1，注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略。 2. 如果解析行协议获得的超级表不存在，则会创建这个超级表（不建议手动创建超级表，不然插入数据可能异常）。 3. 如果解析行协议获得子表不存在，则 Schemaless 会按照步骤 1 或 2 确定的子表名来创建子表。 From 5021d981b8f4c3f3b1b3630b03644e8532355d37 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 22 Jan 2024 11:23:38 +0800 Subject: [PATCH 68/81] fix: fix memory leak --- source/libs/executor/src/timewindowoperator.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 59455840b0..d90fdb38df 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1536,6 +1536,8 @@ void destroySWindowOperatorInfo(void* param) { colDataDestroy(&pInfo->twAggSup.timeWindowData); cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarSupp); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } @@ -1592,7 +1594,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, cleanupExprSupp(&pInfo->scalarSupp); +WindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); From 60b3ac93abe32363b8786bb373a17cc5a0b6b5e8 Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 22 Jan 2024 13:11:11 +0800 Subject: [PATCH 69/81] fix: fix compilation error --- source/libs/executor/src/timewindowoperator.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index d90fdb38df..57c038e75a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1594,8 +1594,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, cleanupExprSupp(&pInfo->scalarSupp); -WindowAgg, NULL, destroySWindowOperatorInfo, + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); From ad170ecb761980bc8f2b0df9bd6eb0ba452ef50e Mon Sep 17 00:00:00 2001 From: Charles Date: Mon, 22 Jan 2024 13:56:07 +0800 Subject: [PATCH 70/81] Delete tests/pytest/arm64_ci_feishu_notice.py --- tests/pytest/arm64_ci_feishu_notice.py | 84 -------------------------- 1 file changed, 84 deletions(-) delete mode 100644 tests/pytest/arm64_ci_feishu_notice.py diff --git a/tests/pytest/arm64_ci_feishu_notice.py b/tests/pytest/arm64_ci_feishu_notice.py deleted file mode 100644 index 9964fb2633..0000000000 --- a/tests/pytest/arm64_ci_feishu_notice.py +++ /dev/null @@ -1,84 +0,0 @@ -import requests -import subprocess - -"""This file is used to send notice to feishu robot with daily arm64 ci test result -""" -group_url = 'https://open.feishu.cn/open-apis/bot/v2/hook/56c333b5-eae9-4c18-b0b6-7e4b7174f5c9' - -def get_msg(text): - return { - "msg_type": "post", - "content": { - "post": { - "zh_cn": { - "title": "ARM64 CI test report", - "content": [ - [{ - "tag": "text", - "text": text - } - ]] - } - } - } - } - -def send_msg(result, testScope, owner, hostname, logDir, others): - text = f'''result: {result} - test scope: {testScope} - owner: {owner} - hostname: {hostname} - log dir: {logDir} - others: {others}\n''' - - json = get_msg(text) - headers = { - 'Content-Type': 'application/json' - } - - req = requests.post(url=group_url, headers=headers, json=json) - inf = req.json() - if "StatusCode" in inf and inf["StatusCode"] == 0: - pass - else: - print(inf) - -def check_build_failed(): - path = "/home/arm64/log/" - res = False - try: - r = subprocess.Popen("ls | wc -l", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path) - stdout, stderr = r.communicate() - if r.returncode != 0: - print("Failed to execute 'ls -l' for path '{}' with error: {}".format(path, stderr)) - else: - if int(stdout) == 0: - res = True - return res - except Exception as ex: - raise Exception("Failed to check build failed with error: {}".format(str(ex))) - -def check_run_case_status(): - path = "/home/arm64/log/*/stat.txt" - r = subprocess.Popen("cat {}".format(path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = r.communicate() - if r.returncode != 0: - print("Failed to execute 'cat {}' with error: {}".format(path, stderr)) - else: - for line in stdout.decode().split("\n"): - if "Failed:" in line: - failed_case_num = int(line.split(":")[1].strip()) - if 0 == failed_case_num: - return "All cases passed" - else: - return stdout.decode() - -def send_notice(): - if check_build_failed(): - send_msg("Build failed", "arm64_ci_test", "charles", "ecs-3e78", "/home/arm64/log", "arm 64 ci build failed") - run_res = check_run_case_status() - if "All cases passed" != run_res: - send_msg(run_res, "arm64_ci_test", "charles", "ecs-3e78", "/home/arm64/log", "arm 64 ci run case failed") - -if __name__ == '__main__': - send_notice() From eb1f10764559596a008bf27d11db50456ce978c9 Mon Sep 17 00:00:00 2001 From: Charles Date: Mon, 22 Jan 2024 13:56:35 +0800 Subject: [PATCH 71/81] Delete tests/system-test/2-query/test_ts4382.py --- tests/system-test/2-query/test_ts4382.py | 531 ----------------------- 1 file changed, 531 deletions(-) delete mode 100644 tests/system-test/2-query/test_ts4382.py diff --git a/tests/system-test/2-query/test_ts4382.py b/tests/system-test/2-query/test_ts4382.py deleted file mode 100644 index 144cd58ff6..0000000000 --- a/tests/system-test/2-query/test_ts4382.py +++ /dev/null @@ -1,531 +0,0 @@ -import random -import itertools -from util.log import * -from util.cases import * -from util.sql import * -from util.sqlset import * -from util import constant -from util.common import * - - -class TDTestCase: - """Verify the jira TS-4382 - """ - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) - # self.dbname = 'db' - # self.stbname = 'st' - # self.ctbname_list = ["ct1", "ct2"] - # self.tag_value_list = ['{"instance":"100"}', '{"instance":"200"}'] - - self.metadata_dic = { - "db_tag_json": { - "supertables": [ - { - "name": "st", - "child_table_num": 2, - "columns": [ - { - "name": "ts", - "type": "timestamp" - }, - { - "name": "col1", - "type": "int" - } - ], - "tags": [ - { - "name": "t1", - "type": "json" - } - ] - } - ] - }, - "db": { - "supertables": [ - { - "name": "st1", - "child_table_num": 2, - "columns": [ - { - "name": "ts", - "type": "timestamp" - }, - { - "name": "col1", - "type": "int" - }, - { - "name": "col2", - "type": "bigint" - }, - { - "name": "col3", - "type": "float" - }, - { - "name": "col4", - "type": "double" - }, - { - "name": "col5", - "type": "bool" - }, - { - "name": "col6", - "type": "binary(16)" - }, - { - "name": "col7", - "type": "nchar(16)" - }, - { - "name": "col8", - "type": "geometry(512)" - }, - { - "name": "col9", - "type": "varbinary(32)" - } - ], - "tags": [ - { - "name": "t1", - "type": "timestamp" - }, - { - "name": "t2", - "type": "int" - }, - { - "name": "t3", - "type": "bigint" - }, - { - "name": "t4", - "type": "float" - }, - { - "name": "t5", - "type": "double" - }, - { - "name": "t6", - "type": "bool" - }, - { - "name": "t7", - "type": "binary(16)" - }, - { - "name": "t8", - "type": "nchar(16)" - }, - { - "name": "t9", - "type": "geometry(512)" - }, - { - "name": "t10", - "type": "varbinary(32)" - } - ] - }, - { - "name": "st2", - "child_table_num": 2, - "columns": [ - { - "name": "ts", - "type": "timestamp" - }, - { - "name": "col1", - "type": "int" - }, - { - "name": "col2", - "type": "bigint" - }, - { - "name": "col3", - "type": "float" - }, - { - "name": "col4", - "type": "double" - }, - { - "name": "col5", - "type": "bool" - }, - { - "name": "col6", - "type": "binary(16)" - }, - { - "name": "col7", - "type": "nchar(16)" - }, - { - "name": "col8", - "type": "geometry(512)" - }, - { - "name": "col9", - "type": "varbinary(32)" - } - ], - "tags": [ - { - "name": "t1", - "type": "timestamp" - }, - { - "name": "t2", - "type": "int" - }, - { - "name": "t3", - "type": "bigint" - }, - { - "name": "t4", - "type": "float" - }, - { - "name": "t5", - "type": "double" - }, - { - "name": "t6", - "type": "bool" - }, - { - "name": "t7", - "type": "binary(16)" - }, - { - "name": "t8", - "type": "nchar(16)" - }, - { - "name": "t9", - "type": "geometry(512)" - }, - { - "name": "t10", - "type": "varbinary(32)" - } - ] - } - ] - } - } - - def prepareData(self): - for db in self.metadata_dic.keys(): - if db == "db_tag_json": - # db - tdSql.execute("create database {};".format(db)) - tdSql.execute("use {};".format(db)) - tdLog.debug("Create database %s" % db) - - # super table - for item in self.metadata_dic[db]["supertables"]: - sql = "create table {} (".format(item["name"]) - for column in item["columns"]: - sql += "{} {},".format(column["name"], column["type"]) - sql = sql[:-1] + ") tags (" - for tag in item["tags"]: - sql += "{} {},".format(tag["name"], tag["type"]) - sql = sql[:-1] + ");" - tdLog.debug(sql) - tdSql.execute(sql) - tdLog.debug("Create super table {}".format(item["name"])) - - # child table - tag_value_list = ['{"instance":"100"}', '{"instance":"200"}'] - for i in range(item["child_table_num"]): - tdSql.execute("create table {} using {} tags('{}');".format("ct" + str(i+1), item["name"], tag_value_list[i])) - tdLog.debug("Create child table {} successfully".format("ct" + str(i+1))) - - # insert data - if i == 0: - tdSql.execute("insert into {} values(now, 1)(now+1s, 2)".format("ct" + str(i+1))) - elif i == 1: - tdSql.execute("insert into {} values(now, null)(now+1s, null)".format("ct" + str(i+1))) - elif db == "db": - # create database db_empty - tdSql.execute("create database db_empty;") - tdSql.execute("use db_empty;") - tdLog.debug("Create database db_empty successfully") - - # super table - for item in self.metadata_dic[db]["supertables"]: - sql = "create table {} (".format(item["name"]) - for column in item["columns"]: - sql += "{} {},".format(column["name"], column["type"]) - sql = sql[:-1] + ") tags (" - for tag in item["tags"]: - sql += "{} {},".format(tag["name"], tag["type"]) - sql = sql[:-1] + ");" - tdLog.debug(sql) - tdSql.execute(sql) - tdLog.debug("Create super table {}".format(item["name"])) - - # child table - tag_value_list = [['2024-01-01 12:00:00.000', 1, 1111111111111, 1.11, 111111.1111, True, 'aaa', 'beijing', 'POINT (3.000000 6.000000)', '0x7661726331'],['2024-01-02 12:00:00.000', 2, 2222222222222, 2.22, 222222.2222, False, 'bbb', 'shanghai', 'LINESTRING (1.000000 1.000000, 2.000000 2.000000, 5.000000 5.000000)', '0x7f829000']] - for i in range(item["child_table_num"]): - sql = "create table {} using {} tags(".format("ct" + (str(i+1) if item["name"] == "st1" else str(i+3)), item["name"]) - for tag in tag_value_list[i]: - if type(tag) == str: - sql += "'{}',".format(tag) - else: - sql += "{},".format(tag) - sql = sql[:-1] + ");" - tdSql.execute(sql) - tdLog.debug("Create child table {} successfully".format("ct" + (str(i+1) if item["name"] == "st1" else str(i+3)))) - - # create database db_with_data - tdSql.execute("create database db_with_data;") - tdSql.execute("use db_with_data;") - tdLog.debug("Create database db_with_data successfully") - - # super table - for item in self.metadata_dic[db]["supertables"]: - sql = "create table {} (".format(item["name"]) - for column in item["columns"]: - sql += "{} {},".format(column["name"], column["type"]) - sql = sql[:-1] + ") tags (" - for tag in item["tags"]: - sql += "{} {},".format(tag["name"], tag["type"]) - sql = sql[:-1] + ");" - tdLog.debug(sql) - tdSql.execute(sql) - tdLog.debug("Create super table {}".format(item["name"])) - - # child table - tag_value_list = [['2024-01-01 12:00:00.000', 1, 1111111111111, 1.11, 111111.1111, True, 'aaa', 'beijing', 'POINT (3.000000 6.000000)', '0x7661726331'],['2024-01-02 12:00:00.000', 2, 2222222222222, 2.22, 222222.2222, False, 'bbb', 'shanghai', 'LINESTRING (1.000000 1.000000, 2.000000 2.000000, 5.000000 5.000000)', '0x7f829000']] - for i in range(item["child_table_num"]): - sql = "create table {} using {} tags(".format("ct" + (str(i+1) if item["name"] == "st1" else str(i+3)), item["name"]) - for tag in tag_value_list[i]: - if type(tag) == str: - sql += "'{}',".format(tag) - else: - sql += "{},".format(tag) - sql = sql[:-1] + ");" - tdSql.execute(sql) - tdLog.debug("Create child table {} successfully".format("ct" + (str(i+1) if item["name"] == "st1" else str(i+3)))) - - # insert into data - start_ts = 1677654000000 # 2023-03-01 15:00:00.000 - sql = "insert into {} values".format("ct" + (str(i+1) if item["name"] == "st1" else str(i+3))) - binary_vlist = ["ccc", "ddd", "eee", "fff"] - nchar_vlist = ["guangzhou", "tianjing", "shenzhen", "hangzhou"] - geometry_vlist = ["POINT (4.0 8.0)", "POINT (3.0 5.0)", "LINESTRING (1.000000 1.000000, 2.000000 2.000000, 5.000000 5.000000)", "POLYGON ((3.000000 6.000000, 5.000000 6.000000, 5.000000 8.000000, 3.000000 8.000000, 3.000000 6.000000))"] - varbinary_vlist = ["0x7661726332", "0x7661726333", "0x7661726334", "0x7661726335"] - st_index = i if item["name"] == "st1" else (i+2) - for i in range(100): - sql += "({}, {}, {}, {}, {}, {}, '{}', '{}', '{}', '{}')".format(start_ts + 1000 * i, str(i+1), str(i+1), str(i+1), str(i+1), True if i % 2 == 0 else False, binary_vlist[st_index % 4], nchar_vlist[st_index % 4], geometry_vlist[st_index % 4], varbinary_vlist[st_index % 4]) - tdSql.execute(sql) - tdLog.debug("Insert into data into child table {} successfully".format("ct" + (str(i+1) if item["name"] == "st1" else str(i+3)))) - - def test_tag_json(self): - tdSql.execute("use db_tag_json;") - sql_list = [ - # super table query with correct tag name of json type - { - "sql": "select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(col1) from st group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), t1->'instance' order by time;", - "result_check": "0.0" - }, - # child table query with incorrect tag name of json type - { - "sql": "select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(col1) from ct1 group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), t1->'name' order by time;", - "result_check": "None" - }, - # child table query with null value - { - "sql": "select ts, avg(col1) from ct2 group by ts, t1->'name' order by ts;", - "result_check": "None" - } - ] - for sql_dic in sql_list: - tdSql.query(sql_dic["sql"]) - tdLog.debug("execute sql: %s" % sql_dic["sql"]) - for item in [row[1] for row in tdSql.queryResult]: - if sql_dic["result_check"] in str(item): - tdLog.debug("Check query result of '{}' successfully".format(sql_dic["sql"])) - break - - def test_db_empty(self): - tdSql.execute("use db_empty;") - table_list = ["st1", "ct1"] - column_list = ["col1", "col2", "col3", "col4", "col5"] - tag_list = ["t2", "t3", "t4", "t5", "t6"] - operator_list = ["+", "-", "*", "/"] - fun_list = ["avg", "count", "sum", "spread"] - - # two columns with arithmetic operation - for table in table_list: - for columns in list(itertools.combinations(column_list + tag_list, 2)): - operator = random.choice(operator_list) - sql = "select ({} {} {}) as total from {};".format(columns[0], operator, columns[1], table) - tdSql.query(sql) - tdSql.checkRows(0) - - # aggregation function - for table in table_list: - for columns in list(itertools.combinations(column_list[:-1] + tag_list[:-1], 2)): - fun = random.sample(fun_list, 2) - sql = "select ({}({}) + {}({})) as total from {};".format(fun[0], columns[0], fun[1], columns[1], table) - tdSql.query(sql) - if "count" in fun: - # default config 'countAlwaysReturnValue' as 0 - tdSql.checkRows(1) - else: - tdSql.checkRows(0) - - # join - table_list = ["st1", "st2", "ct1", "ct2", "ct3", "ct4"] - column_list = ["col1", "col2", "col3", "col4", "col5"] - tag_list = ["t2", "t3", "t4", "t5", "t6"] - where_list = ["col1 > 100", "col2 < 237883294", "col3 >= 163.23", "col4 <= 674324.2374898237", "col5=true", "col6='aaa'", - "col7='beijing'", "col8!='POINT (3.000000 6.000000)'", "col9='0x7661726331'"] - for table in list(itertools.combinations(table_list,2)): - where = random.choice(where_list) - column = random.choice(column_list) - tag = random.choice(tag_list) - sql = "select ({} + {}) total from {} join {} on {} where {};".format(table[0] + "." + column, table[1] + "." + tag, table[0], table[1], table[0]+ ".ts=" + table[1] + ".ts", table[0] + "." + where) - tdSql.query(sql) - tdSql.checkRows(0) - - # group by - value_fun_list = ["sum(col1+col2)", "avg(col3+col4)", "count(col6+col7)", "stddev(col2+col4)", "spread(col2+col3)"] - group_by_list = ["tbname", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9", "t10"] - for table in table_list: - value_fun = random.choice(value_fun_list) - where = random.choice(where_list) - group_by = random.choice(group_by_list) - sql = "select {} from {} where {} group by {};".format(value_fun, table, where, group_by) - tdSql.query(sql) - # default config 'countAlwaysReturnValue' as 0 - if "count" in value_fun and "st" in table: - tdSql.checkRows(2) - elif "count" in value_fun and "ct" in table: - tdSql.checkRows(1) - else: - tdSql.checkRows(0) - - # window query - for table in table_list: - tag = random.choice(tag_list) - if "st" in table: - sql = "select _wstart, {}, avg(col3+col4) from {} where ts between '2024-03-01' and '2024-03-02' partition by {} interval(10s) sliding(5s) fill(linear);".format(tag, table, tag) - elif "ct" in table: - sql = "select _wstart, sum(col1+col2) from {} where ts between '2024-03-01' and '2024-03-02' partition by {} interval(10s) sliding(5s) fill(next);".format(table, tag) - tdSql.query(sql) - tdSql.checkRows(0) - - # nested query - for table in table_list: - sql_list = [ - "select (col1 + col2) from (select sum(col1) as col1, avg(col2) as col2 from {} where col1 > 100 and ts between '2024-03-01' and '2024-03-02' group by tbname);".format(table), - "select last(ts), avg(col2 - col3) from (select first(ts) as ts, sum(col2) as col2, last(col3) as col3 from {} where col9 != 'abc' partition by tbname interval(10s) sliding(5s));".format(table), - "select elapsed(ts, 1s), sum(c1 + c2) from (select * from (select ts, (col1+col2) as c1, (col3 * col4) as c2, tbname from {} where col1 > 100 and ts between '2024-03-01' and '2024-03-02')) group by tbname;".format(table) - ] - for sql in sql_list: - tdSql.query(sql) - tdSql.checkRows(0) - - # drop column/tag - del_column_tag_list = ["col1", "t1"] - error_sql_list = [ - "select first(t1), sum(col1) from st1 group by tbname;", - "select last(ts), avg(col1) from st1 group by tbname;", - "select count(col1) from (select * from st1 where ts between '2024-03-01' and '2024-03-02' and col1 > 100) group by tbname;", - ] - for item in del_column_tag_list: - if "col" in item: - sql = "alter table st1 drop column {};".format(item) - elif "t" in item: - sql = "alter table st1 drop tag {};".format(item) - tdSql.execute(sql) - tdLog.debug("Delete {} successfully".format(str(del_column_tag_list))) - - for table in table_list: - for sql in error_sql_list: - tdSql.error(sql) - - # modify column for common table - tdSql.execute("create table t1 (ts timestamp, col1 int, col2 bigint, col3 float, col4 double, col5 bool, col6 binary(16), col7 nchar(16), col8 geometry(512), col9 varbinary(32));") - tdSql.execute("insert into t1 values(now, 1, 1111111111111, 1.11, 111111.1111, True, 'aaa', 'beijing', 'POINT (3.000000 6.000000)', '0x7661726331');") - tdSql.execute("alter table t1 rename column col1 col11;") - tdSql.error("select col1 from t1 where ts <= now and col3=1.11;") - tdSql.query("select col11 from t1 where ts <= now and col3=1.11;") - assert(len(tdSql.queryResult) == 1 and tdSql.queryResult[0][0] == 1) - - def test_db_with_data(self): - tdSql.execute("use db_with_data;") - - sql_list = [ - "select pow(col1, null) from st1 where ts > now;", - "select pow(null, col1) from st1 where ts > now;", - "select log(null, col2) from st1 where col1 > 1000;", - "select log(col2, null) from st1 where col1 > 1000;", - "select avg(col1 + t2) from ct1 where ts between '2025-03-01' and '2025-03-02' and t2 < 0;", - "select char_length(col6) from st1 where ts > now;", - "select concat(col6, col7) from st1 where ts > now;", - "select char_length(concat(col6, col7)) from st1 where ts > now;", - "select rtrim(ltrim(concat(col6, col7))) from st1 where ts > now;", - "select lower(rtrim(ltrim(concat(col6, col7)))) from st1 where ts > now;", - "select upper(rtrim(ltrim(concat(col6, col7)))) from st1 where ts > now;", - "select substr(rtrim(ltrim(concat(col6, col7))), 1, 10) from st1 where ts > now;", - "select avg(col1 - col2) as v from st1 where ts between '2022-03-01' and '2022-03-02';", - "select avg(col1 * col3) as v from st1 where ts between '2022-03-01' and '2022-03-02' and col1 > 100 group by tbname;", - "select sum(col1 / col4) as cv, avg(t2 + t3) as tv from st1 where ts between '2022-03-01' and '2022-03-02' and col1 > 100 group by tbname;", - "select sum(v1+v2) from (select first(ts) as time, avg(col1+col2) as v1, max(col3) as v2 from st1 where ts > now group by (col1+col2) order by (col1+col2));", - "select first(ts), count(*), avg(col2 * t3) from (select ts, col1, col2, col3, t1, t2, t3, tbname from st1 where ts between '2022-03-01' and '2022-03-02' and col1 > 100) group by tbname;", - "select cast(t8 as nchar(32)), sum(col1), avg(col2) from st1 where ts > now group by cast(t8 as nchar(32));", - "select to_char(time, 'yyyy-mm-dd'), sum(v2 - v1) from (select first(ts) as time, avg(col2 + col3) as v1, max(col4) as v2 from st1 where ts < now group by (col2+col3) order by (col2+col3)) where time > now group by to_char(time, 'yyyy-mm-dd');", - "select count(time) * sum(v) from (select to_iso8601(ts, '+00:00') as time, abs(col1+col2) as v, tbname from st1 where ts between '2023-03-01' and '2023-03-02' and col1 > 100) group by tbname;", - "select avg(v) from (select apercentile(col1, 50) as v from st1 where ts between '2023-03-01' and '2023-03-02' group by tbname) where v > 50;", - ] - for sql in sql_list: - tdSql.query(sql) - tdSql.checkRows(0) - - sql_list_with_res = [ - { - "sql": "select total / v from (select elapsed(ts, 1s) as v, sum(col1) as total from st1 where ts between '2023-03-01' and '2023-03-02' interval(10s) fill(next));", - "res": [8641, (11,)] - }, - { - "sql": "select to_char(time, 'yyyy-mm-dd'), sum(v2 - v1) from (select first(ts) as time, avg(col2 + col3) as v1, max(col4) as v2 from st1 where ts < now group by (col2+col3) order by (col2+col3)) group by to_char(time, 'yyyy-mm-dd');", - "res": [1, ('2023-03-01', -5050,)] - }, - { - "sql": "select avg(v) from (select apercentile(col1, 50) as v from st1 where ts between '2023-03-01' and '2023-03-02' group by tbname) group by (50 -v);", - "res": [1, (50,)] - } - ] - for sql in sql_list_with_res: - tdSql.query(sql["sql"]) - assert(len(tdSql.queryResult) == sql["res"][0] and tdSql.queryResult[0][0] == sql["res"][1][0]) - - def run(self): - self.prepareData() - self.test_tag_json() - self.test_db_empty() - self.test_db_with_data() - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) From 5d58c27f56e11b2dc00b3b3a84157e894292be4f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jan 2024 14:03:11 +0800 Subject: [PATCH 72/81] fix: join subquery timestamp order mis-match issue --- source/libs/planner/src/planOptimizer.c | 128 ++++++++++++++++++++++++ tests/parallel_test/cases.task | 1 + tests/script/tsim/query/join_order.sim | 51 ++++++++++ 3 files changed, 180 insertions(+) create mode 100644 tests/script/tsim/query/join_order.sim diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index dfb9343cc8..231e2fa32f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -338,6 +338,7 @@ static void scanPathOptSetScanOrder(EScanOrder scanOrder, SScanLogicNode* pScan) if (pScan->sortPrimaryKey || pScan->scanSeq[0] > 1 || pScan->scanSeq[1] > 1) { return; } + pScan->node.outputTsOrder = scanOrder; switch (scanOrder) { case SCAN_ORDER_ASC: pScan->scanSeq[0] = 1; @@ -1450,6 +1451,132 @@ static int32_t sortPrimaryKeyOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLo return sortPrimaryKeyOptimizeImpl(pCxt, pLogicSubplan, pSort); } +static int32_t sortForJoinOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SJoinLogicNode* pJoin) { + SLogicNode* pLeft = (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0); + SLogicNode* pRight = (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1); + SScanLogicNode* pScan = NULL; + + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pLeft) && ((SScanLogicNode*)pLeft)->node.outputTsOrder != SCAN_ORDER_BOTH) { + pScan = (SScanLogicNode*)pLeft; + } else if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pRight) && ((SScanLogicNode*)pRight)->node.outputTsOrder != SCAN_ORDER_BOTH) { + pScan = (SScanLogicNode*)pRight; + } + + if (NULL != pScan) { + switch (pScan->node.outputTsOrder) { + case SCAN_ORDER_ASC: + pScan->scanSeq[0] = 0; + pScan->scanSeq[1] = 1; + pScan->node.outputTsOrder = SCAN_ORDER_DESC; + goto _return; + case SCAN_ORDER_DESC: + pScan->scanSeq[0] = 1; + pScan->scanSeq[1] = 0; + pScan->node.outputTsOrder = SCAN_ORDER_ASC; + goto _return; + default: + break; + } + } + + if (QUERY_NODE_OPERATOR != nodeType(pJoin->pPrimKeyEqCond)) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + + bool res = false; + SOperatorNode* pOp = (SOperatorNode*)pJoin->pPrimKeyEqCond; + if (QUERY_NODE_COLUMN != nodeType(pOp->pLeft) || QUERY_NODE_COLUMN != nodeType(pOp->pRight)) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + + SNode* pOrderByNode = NULL; + SSHashObj* pLeftTables = NULL; + collectTableAliasFromNodes(nodesListGetNode(pJoin->node.pChildren, 0), &pLeftTables); + + if (NULL != tSimpleHashGet(pLeftTables, ((SColumnNode*)pOp->pLeft)->tableAlias, strlen(((SColumnNode*)pOp->pLeft)->tableAlias))) { + pOrderByNode = pOp->pLeft; + } else if (NULL != tSimpleHashGet(pLeftTables, ((SColumnNode*)pOp->pRight)->tableAlias, strlen(((SColumnNode*)pOp->pRight)->tableAlias))) { + pOrderByNode = pOp->pRight; + } + + tSimpleHashCleanup(pLeftTables); + + if (NULL == pOrderByNode) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + + SSortLogicNode* pSort = (SSortLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSort->node.outputTsOrder = (ORDER_ASC == pLeft->outputTsOrder) ? ORDER_DESC : ORDER_ASC; + pSort->groupSort = false; + SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOrder) { + nodesDestroyNode((SNode *)pSort); + return TSDB_CODE_OUT_OF_MEMORY; + } + + nodesListMakeAppend(&pSort->pSortKeys, (SNode*)pOrder); + pOrder->order = (ORDER_ASC == pLeft->outputTsOrder) ? ORDER_DESC : ORDER_ASC; + pOrder->pExpr = nodesCloneNode(pOrderByNode); + pOrder->nullOrder = (ORDER_ASC == pOrder->order) ? NULL_ORDER_FIRST : NULL_ORDER_LAST; + if (!pOrder->pExpr) { + nodesDestroyNode((SNode *)pSort); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pLeft->pParent = (SLogicNode*)pSort; + nodesListMakeAppend(&pSort->node.pChildren, (SNode*)pLeft); + pJoin->node.pChildren->pHead->pNode = (SNode*)pSort; + pSort->node.pParent = (SLogicNode*)pJoin;; + +_return: + + pCxt->optimized = true; + + return TSDB_CODE_SUCCESS; +} + + +static bool sortForJoinOptMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_JOIN != nodeType(pNode)) { + return false; + } + + SJoinLogicNode* pJoin = (SJoinLogicNode*)pNode; + if (pNode->pChildren->length != 2 || !pJoin->hasSubQuery || pJoin->isLowLevelJoin) { + return false; + } + + SLogicNode* pLeft = (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0); + SLogicNode* pRight = (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1); + + if (ORDER_ASC != pLeft->outputTsOrder && ORDER_DESC != pLeft->outputTsOrder) { + return false; + } + if (ORDER_ASC != pRight->outputTsOrder && ORDER_DESC != pRight->outputTsOrder) { + return false; + } + + if (pLeft->outputTsOrder == pRight->outputTsOrder) { + return false; + } + + return true; +} + + +static int32_t sortForJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SJoinLogicNode* pJoin = (SJoinLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, sortForJoinOptMayBeOptimized); + if (NULL == pJoin) { + return TSDB_CODE_SUCCESS; + } + return sortForJoinOptimizeImpl(pCxt, pLogicSubplan, pJoin); +} + + static bool smaIndexOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode) || NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) || @@ -4197,6 +4324,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "StableJoin", .optimizeFunc = stableJoinOptimize}, {.pName = "sortNonPriKeyOptimize", .optimizeFunc = sortNonPriKeyOptimize}, {.pName = "SortPrimaryKey", .optimizeFunc = sortPrimaryKeyOptimize}, + {.pName = "SortForjoin", .optimizeFunc = sortForJoinOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaIndexOptimize}, {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}, {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0687a3271c..8d9794b0dd 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1108,6 +1108,7 @@ e ,,y,script,./test.sh -f tsim/query/udf_with_const.sim ,,y,script,./test.sh -f tsim/query/join_interval.sim ,,y,script,./test.sh -f tsim/query/join_pk.sim +,,y,script,./test.sh -f tsim/query/join_order.sim ,,y,script,./test.sh -f tsim/query/count_spread.sim ,,y,script,./test.sh -f tsim/query/unionall_as_table.sim ,,y,script,./test.sh -f tsim/query/multi_order_by.sim diff --git a/tests/script/tsim/query/join_order.sim b/tests/script/tsim/query/join_order.sim new file mode 100644 index 0000000000..bd772ff407 --- /dev/null +++ b/tests/script/tsim/query/join_order.sim @@ -0,0 +1,51 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql drop database if exists db1; +sql create database db1 vgroups 1; +sql use db1; +sql create stable sta (ts timestamp, col1 int) tags(t1 int); +sql create table tba1 using sta tags(1); + +sql insert into tba1 values ('2023-11-17 16:29:00', 1); +sql insert into tba1 values ('2023-11-17 16:29:02', 3); +sql insert into tba1 values ('2023-11-17 16:29:03', 4); +sql insert into tba1 values ('2023-11-17 16:29:04', 5); + + +sql select a.*,b.* from tba1 a, (select * from tba1 order by ts) b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi +sql select a.*,b.* from (select * from tba1 order by ts) a, tba1 b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi +sql select a.*,b.* from tba1 a, (select * from tba1 order by ts desc) b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi +sql select a.*,b.* from (select * from tba1 order by ts desc) a, tba1 b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi +sql select a.*,b.* from (select * from tba1 order by ts) a, (select * from tba1 order by ts) b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi +sql select a.*,b.* from (select * from tba1 order by ts desc) a, (select * from tba1 order by ts desc) b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi +sql select a.*,b.* from (select * from tba1 order by ts) a, (select * from tba1 order by ts desc) b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi +sql select a.*,b.* from (select * from tba1 order by ts desc) a, (select * from tba1 order by ts) b where a.ts=b.ts; +if $rows != 4 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From a5656ad4b90f197ed93b209fb031b904ff92e5e7 Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Mon, 22 Jan 2024 14:10:52 +0800 Subject: [PATCH 73/81] fix: fix openssl download url --- cmake/ssl_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index fe176498a5..4778ea76e0 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -1,6 +1,6 @@ # openssl ExternalProject_Add(openssl - URL https://www.openssl.org/source/openssl-3.1.3.tar.gz + URL https://github.com/openssl/openssl/releases/download/openssl-3.1.3/openssl-3.1.3.tar.gz URL_HASH SHA256=f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" From ec16e79b395cb2ded48b49fc61e7cb0178e789f7 Mon Sep 17 00:00:00 2001 From: menshibin Date: Mon, 22 Jan 2024 14:41:48 +0800 Subject: [PATCH 74/81] add srvCtl cluster support --- tests/army/community/cluster/incSnapshot.py | 15 ++++++++------- tests/army/frame/server/cluster.py | 1 + tests/army/frame/server/dnodes.py | 9 +++++++++ tests/army/frame/srvCtl.py | 18 +++++++++++++++++- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index b1b53e65bd..d309bae72c 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -9,11 +9,12 @@ import time from frame.log import * from frame.cases import * from frame.sql import * +from frame.srvCtl import * from frame.caseBase import * from frame import * from frame.autogen import * -from frame.server.dnodes import * -from frame.server.cluster import * +# from frame.server.dnodes import * +# from frame.server.cluster import * class TDTestCase(TBase): @@ -34,7 +35,7 @@ class TDTestCase(TBase): autoGen.create_child(self.stb, "d", self.childtable_count) autoGen.insert_data(1000) tdSql.execute(f"flush database {self.db}") - clusterDnodes.stoptaosd(3) + sc.dnodeStop(3) # clusterDnodes.stoptaosd(1) # clusterDnodes.starttaosd(3) # time.sleep(5) @@ -56,7 +57,7 @@ class TDTestCase(TBase): # break self.snapshotAgg() time.sleep(10) - clusterDnodes.stopAll() + sc.dnodeStopAll() for i in range(1, 4): path = clusterDnodes.getDnodeDir(i) dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") @@ -66,9 +67,9 @@ class TDTestCase(TBase): tdLog.debug("delete dir: %s " % (dnodesRootDir)) self.remove_directory(os.path.join(dir, "wal")) - clusterDnodes.starttaosd(1) - clusterDnodes.starttaosd(2) - clusterDnodes.starttaosd(3) + sc.dnodeStart(1) + sc.dnodeStart(2) + sc.dnodeStart(3) sql = "show vnodes;" time.sleep(10) while True: diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py index 309ca3bbd4..d514bfd9d5 100644 --- a/tests/army/frame/server/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -22,6 +22,7 @@ class ClusterDnodes(TDDnodes): def init(self, dnodes_lists, deployPath, masterIp): self.dnodes = dnodes_lists # dnode must be TDDnode instance super(ClusterDnodes, self).init(deployPath, masterIp) + self.model = "cluster" clusterDnodes = ClusterDnodes() diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index ad8a336857..7a37badd06 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -47,6 +47,7 @@ class TDDnodes: self.valgrind = 0 self.asan = False self.killValgrind = 0 + self.model = "dnode" def init(self, path, remoteIP = ""): binPath = self.dnodes[0].getPath() + "/../../../" @@ -268,6 +269,14 @@ class TDDnodes: def getAsan(self): return self.asan + def getModel(self): + return self.model + + def getDnodeCfgPath(self, index): + self.check(index) + return self.dnodes[index - 1].cfgPath + + def setLevelDisk(self, level, disk): for i in range(len(self.dnodes)): self.dnodes[i].level = int(level) diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index c01ea33578..091856056b 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -18,6 +18,7 @@ import datetime from frame.server.dnode import * from frame.server.dnodes import * +from frame.server.cluster import * class srvCtl: @@ -34,19 +35,32 @@ class srvCtl: # start def dnodeStart(self, idx): + if clusterDnodes.getModel() == 'cluster': + return clusterDnodes.starttaosd(idx) + return tdDnodes.starttaosd(idx) # stop def dnodeStop(self, idx): + if clusterDnodes.getModel() == 'cluster': + return clusterDnodes.stoptaosd(idx) + return tdDnodes.stoptaosd(idx) + def dnodeStopAll(self): + if clusterDnodes.getModel() == 'cluster': + return clusterDnodes.stopAll() + return tdDnodes.stopAll() # # about path # # get cluster root path like /root/TDinternal/sim/ def clusterRootPath(self): + if clusterDnodes.getModel() == 'cluster': + return clusterDnodes.getDnodesRootDir() + return tdDnodes.getDnodesRootDir() # return dnode data files list @@ -60,7 +74,9 @@ class srvCtl: # taos.cfg position def dnodeCfgPath(self, idx): - return tdDnodes.dnodes[idx-1].cfgPath + if clusterDnodes.getModel() == 'cluster': + return clusterDnodes.getDnodeCfgPath(idx) + return tdDnodes.getDnodeCfgPath(idx) sc = srvCtl() \ No newline at end of file From 25b3ce511f9dba25a9358e15797950be2dd0ae78 Mon Sep 17 00:00:00 2001 From: menshibin Date: Mon, 22 Jan 2024 16:03:04 +0800 Subject: [PATCH 75/81] modify dnodes default single model --- tests/army/frame/server/dnodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 7a37badd06..cd2b89acbd 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -47,7 +47,7 @@ class TDDnodes: self.valgrind = 0 self.asan = False self.killValgrind = 0 - self.model = "dnode" + self.model = "single" def init(self, path, remoteIP = ""): binPath = self.dnodes[0].getPath() + "/../../../" From c939ce4c3de8cde0d5684ef4ee47c5e7da82d10d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jan 2024 17:03:45 +0800 Subject: [PATCH 76/81] fix: scheduler save execution result issue --- source/libs/scheduler/src/schJob.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index b565619e75..6e312f0e6f 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -386,10 +386,13 @@ _return: int32_t schDumpJobExecRes(SSchJob *pJob, SExecResult *pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; + + SCH_LOCK(SCH_WRITE, &pJob->resLock); pRes->res = pJob->execRes.res; pRes->msgType = pJob->execRes.msgType; pRes->numOfBytes = pJob->execRes.numOfBytes; pJob->execRes.res = NULL; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); SCH_JOB_DLOG("execRes dumped, code: %s", tstrerror(pRes->code)); From cbe34ad76e8e86d7532ff41958ffa8315d5ef18a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 22 Jan 2024 17:11:15 +0800 Subject: [PATCH 77/81] fix: mac compile error --- source/libs/planner/src/planOptimizer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 231e2fa32f..af9ec93f65 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -338,7 +338,7 @@ static void scanPathOptSetScanOrder(EScanOrder scanOrder, SScanLogicNode* pScan) if (pScan->sortPrimaryKey || pScan->scanSeq[0] > 1 || pScan->scanSeq[1] > 1) { return; } - pScan->node.outputTsOrder = scanOrder; + pScan->node.outputTsOrder = (SCAN_ORDER_ASC == scanOrder) ? ORDER_ASC : ORDER_DESC; switch (scanOrder) { case SCAN_ORDER_ASC: pScan->scanSeq[0] = 1; @@ -1467,12 +1467,12 @@ static int32_t sortForJoinOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pL case SCAN_ORDER_ASC: pScan->scanSeq[0] = 0; pScan->scanSeq[1] = 1; - pScan->node.outputTsOrder = SCAN_ORDER_DESC; + pScan->node.outputTsOrder = ORDER_DESC; goto _return; case SCAN_ORDER_DESC: pScan->scanSeq[0] = 1; pScan->scanSeq[1] = 0; - pScan->node.outputTsOrder = SCAN_ORDER_ASC; + pScan->node.outputTsOrder = ORDER_ASC; goto _return; default: break; From 48bb6c2bbb001a7eb97e5e605bfbde931b661273 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 21 Jan 2024 15:06:46 +0800 Subject: [PATCH 78/81] fix: time pseudo column used illegally --- include/util/taoserror.h | 1 + source/libs/planner/src/planPhysiCreater.c | 40 +++++++++++++++++++++- source/util/src/terror.c | 1 + 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b5389e60d3..5e27358166 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -521,6 +521,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) // #define TSDB_CODE_QRY_INVALID_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0710) // 2.x // #define TSDB_CODE_QRY_RESULT_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0711) // 2.x +#define TSDB_CODE_QRY_INVALID_WINDOW_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0712) #define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0720) #define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0721) #define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0722) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 21c637116f..301ba4fceb 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1722,6 +1722,39 @@ static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return code; } +static EDealRes collectWindowsPseudocolumns(SNode* pNode, void* pContext) { + SNodeList* pCols = (SNodeList*)pContext; + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || + FUNCTION_TYPE_WDURATION == pFunc->funcType) { + nodesListStrictAppend(pCols, nodesCloneNode(pNode)); + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t checkWindowsConditonValid(SWindowLogicNode* pWindowLogicNode) { + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pCols = nodesMakeList(); + if (NULL == pCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + nodesWalkExpr(pWindowLogicNode->pStartCond, collectWindowsPseudocolumns, pCols); + if (pCols->length > 0) { + code = TSDB_CODE_QRY_INVALID_WINDOW_CONDITION; + } + if (TSDB_CODE_SUCCESS == code) { + nodesWalkExpr(pWindowLogicNode->pEndCond, collectWindowsPseudocolumns, pCols); + if (pCols->length > 0) { + code = TSDB_CODE_QRY_INVALID_WINDOW_CONDITION; + } + } + + nodesDestroyList(pCols); + return code; +} + static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SEventWinodwPhysiNode* pEvent = (SEventWinodwPhysiNode*)makePhysiNode( @@ -1731,8 +1764,13 @@ static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return TSDB_CODE_OUT_OF_MEMORY; } + int32_t code = checkWindowsConditonValid(pWindowLogicNode); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); - int32_t code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pStartCond, &pEvent->pStartCond); + code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pStartCond, &pEvent->pStartCond); if (TSDB_CODE_SUCCESS == code) { code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pEndCond, &pEvent->pEndCond); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 79b9e9bbed..9aa1b97190 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -426,6 +426,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JSON_IN_GROUP_ERROR, "Json not support in g TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_NOT_EXIST, "Job not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_QWORKER_QUIT, "Vnode/Qnode is quitting") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR, "Geometry not support in this operator") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_WINDOW_CONDITION, "The time pseudo column is illegally used in the condition of the event window.") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR, "Executor internal error") // grant From 119d64ecdc6b91c6383bc8fc5ad4cd580b06f473 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 22 Jan 2024 11:11:06 +0800 Subject: [PATCH 79/81] add test case --- tests/system-test/2-query/group_partition.py | 32 +++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/group_partition.py b/tests/system-test/2-query/group_partition.py index e228351f0e..a20b124c33 100644 --- a/tests/system-test/2-query/group_partition.py +++ b/tests/system-test/2-query/group_partition.py @@ -168,8 +168,37 @@ class TDTestCase: tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9;") tdSql.checkRows(nonempty_tb_num) + def test_event_window(self, nonempty_tb_num): + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and 1=1;") + tdSql.checkRows(nonempty_tb_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and 1=0;") + tdSql.checkRows(0) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and tbname='sub_{self.stable}_0';") + tdSql.checkRows(1) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and t2=0;") + tdSql.checkRows(1) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _rowts>0;") + tdSql.checkRows(nonempty_tb_num) + + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _qstart>0;") + tdSql.checkRows(0) + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _qstart<0;") + tdSql.checkRows(0) + tdSql.query(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _qstart<_qend;") + tdSql.checkRows(0) + + tdSql.error(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _wstart= 0 end with c2 = 9 and _wstart - q_start > 0;") + tdSql.error(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _irowts>0;") + tdSql.error(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 and _wduration > 5s end with c2 = 9;") + tdSql.error(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _wstart > 1299845454;") + tdSql.error(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and _wduration + 1s > 5s;") + tdSql.error(f"select tbname, count(*) from {self.dbname}.{self.stable} partition by tbname event_window start with c1 >= 0 end with c2 = 9 and count(*) > 10;") - def test_error(self): tdSql.error(f"select * from {self.dbname}.{self.stable} group by t2") tdSql.error(f"select t2, count(*) from {self.dbname}.{self.stable} group by t2 where t2 = 1") @@ -197,6 +226,7 @@ class TDTestCase: self.test_multi_group_key(self.tb_nums, nonempty_tb_num) self.test_multi_agg(self.tb_nums, nonempty_tb_num) self.test_window(nonempty_tb_num) + self.test_event_window(nonempty_tb_num) ## test old version before changed # self.test_groupby('group', 0, 0) From 6ef1d8b0cb460229e8d1e33764beddea8d1aa1e2 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 22 Jan 2024 17:16:55 +0800 Subject: [PATCH 80/81] checkout pseudo columns --- source/libs/parser/src/parTranslater.c | 40 ++++++++++++++++++++++ source/libs/planner/src/planPhysiCreater.c | 40 +--------------------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 70e4744644..f8d1573df4 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3981,6 +3981,42 @@ static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSe return TSDB_CODE_SUCCESS; } +static EDealRes collectWindowsPseudocolumns(SNode* pNode, void* pContext) { + SNodeList* pCols = (SNodeList*)pContext; + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || + FUNCTION_TYPE_WDURATION == pFunc->funcType) { + nodesListStrictAppend(pCols, nodesCloneNode(pNode)); + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t checkWindowsConditonValid(SNode* pNode) { + int32_t code = TSDB_CODE_SUCCESS; + if(QUERY_NODE_EVENT_WINDOW != nodeType(pNode)) return code; + + SEventWindowNode* pEventWindowNode = (SEventWindowNode*)pNode; + SNodeList* pCols = nodesMakeList(); + if (NULL == pCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + nodesWalkExpr(pEventWindowNode->pStartCond, collectWindowsPseudocolumns, pCols); + if (pCols->length > 0) { + code = TSDB_CODE_QRY_INVALID_WINDOW_CONDITION; + } + if (TSDB_CODE_SUCCESS == code) { + nodesWalkExpr(pEventWindowNode->pEndCond, collectWindowsPseudocolumns, pCols); + if (pCols->length > 0) { + code = TSDB_CODE_QRY_INVALID_WINDOW_CONDITION; + } + } + + nodesDestroyList(pCols); + return code; +} + static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; @@ -3994,6 +4030,10 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = translateSpecificWindow(pCxt, pSelect); } + code = checkWindowsConditonValid(pSelect->pWindow); + if (TSDB_CODE_SUCCESS != code) { + return code; + } return code; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 301ba4fceb..21c637116f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1722,39 +1722,6 @@ static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return code; } -static EDealRes collectWindowsPseudocolumns(SNode* pNode, void* pContext) { - SNodeList* pCols = (SNodeList*)pContext; - if (QUERY_NODE_FUNCTION == nodeType(pNode)) { - SFunctionNode* pFunc = (SFunctionNode*)pNode; - if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || - FUNCTION_TYPE_WDURATION == pFunc->funcType) { - nodesListStrictAppend(pCols, nodesCloneNode(pNode)); - } - } - return DEAL_RES_CONTINUE; -} - -static int32_t checkWindowsConditonValid(SWindowLogicNode* pWindowLogicNode) { - int32_t code = TSDB_CODE_SUCCESS; - SNodeList* pCols = nodesMakeList(); - if (NULL == pCols) { - return TSDB_CODE_OUT_OF_MEMORY; - } - nodesWalkExpr(pWindowLogicNode->pStartCond, collectWindowsPseudocolumns, pCols); - if (pCols->length > 0) { - code = TSDB_CODE_QRY_INVALID_WINDOW_CONDITION; - } - if (TSDB_CODE_SUCCESS == code) { - nodesWalkExpr(pWindowLogicNode->pEndCond, collectWindowsPseudocolumns, pCols); - if (pCols->length > 0) { - code = TSDB_CODE_QRY_INVALID_WINDOW_CONDITION; - } - } - - nodesDestroyList(pCols); - return code; -} - static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SEventWinodwPhysiNode* pEvent = (SEventWinodwPhysiNode*)makePhysiNode( @@ -1764,13 +1731,8 @@ static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return TSDB_CODE_OUT_OF_MEMORY; } - int32_t code = checkWindowsConditonValid(pWindowLogicNode); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); - code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pStartCond, &pEvent->pStartCond); + int32_t code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pStartCond, &pEvent->pStartCond); if (TSDB_CODE_SUCCESS == code) { code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pEndCond, &pEvent->pEndCond); } From bde8c14b55eae9d91b960769208a3042c08daf7e Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 22 Jan 2024 20:54:24 +0800 Subject: [PATCH 81/81] fix: error code is overwritten --- source/libs/parser/src/parTranslater.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index f8d1573df4..23396a499d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4030,9 +4030,8 @@ static int32_t translateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = translateSpecificWindow(pCxt, pSelect); } - code = checkWindowsConditonValid(pSelect->pWindow); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (TSDB_CODE_SUCCESS == code) { + code = checkWindowsConditonValid(pSelect->pWindow); } return code; }