From 1e1951e86d0bebf4f6807b4e00c1e9126a8821f8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 19 Jan 2024 17:51:55 +0800 Subject: [PATCH 1/7] fix: add distributed total information --- tests/army/community/query/query_basic.py | 14 ++++++++++++ tests/army/frame/caseBase.py | 27 +++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 90916a1c6c..13b37ef253 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -76,12 +76,26 @@ class TDTestCase(TBase): sql2 = "select bi from stb where bi is not null order by bi desc limit 10;" self.checkSameResult(sql1, sql2) + # distributed expect values + expects = { + "Block_Rows" : 6*100000, + "Total_Tables" : 6, + "Total_Vgroups" : 3 + } + self.waitTransactionZero() + reals = self.getDistributed(self.stb) + for k in expects.keys(): + v = expects[k] + if int(reals[k]) != v: + tdLog.exit(f"distribute {k} expect: {v} real: {reals[k]}") + # run def run(self): tdLog.debug(f"start to excute {__file__}") # insert data self.insertData() + self.flushDb() # check insert data correct self.checkInsertCorrect() diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index b1500b69e1..5715867faf 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -229,9 +229,9 @@ class TBase: # # get vgroups - def getVGroup(self, db_name): + def getVGroup(self, dbName): vgidList = [] - sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'" + sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{dbName}'" res = tdSql.getResult(sql) rows = len(res) for i in range(rows): @@ -239,6 +239,29 @@ class TBase: return vgidList + # get distributed rows + def getDistributed(self, tbName): + sql = f"show table distributed {tbName}" + tdSql.query(sql) + dics = {} + i = 0 + for i in range(tdSql.getRows()): + row = tdSql.getData(i, 0) + #print(row) + row = row.replace('[', '').replace(']', '') + #print(row) + items = row.split(' ') + #print(items) + for item in items: + #print(item) + v = item.split('=') + #print(v) + if len(v) == 2: + dics[v[0]] = v[1] + if i > 5: + break + print(dics) + return dics # From 95975801d52c1e4631b8a39b26285bdea3d148d1 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 13:42:42 +0800 Subject: [PATCH 2/7] feat: add check max and ts window correctness --- tests/army/community/query/query_basic.json | 2 +- tests/army/community/query/query_basic.py | 142 +++++++++++++++++++- 2 files changed, 140 insertions(+), 4 deletions(-) diff --git a/tests/army/community/query/query_basic.json b/tests/army/community/query/query_basic.json index c6a3482cd4..d8d47266f9 100644 --- a/tests/army/community/query/query_basic.json +++ b/tests/army/community/query/query_basic.json @@ -32,7 +32,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 30000, - "start_timestamp":"2023-10-01 10:00:00", + "start_timestamp":1700000000000, "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 13b37ef253..eb332b8c71 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -34,23 +34,158 @@ class TDTestCase(TBase): "querySmaOptimize": "1" } + def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run jfile = etool.curFile(__file__, "query_basic.json") - etool.benchMark(json=jfile) + etool.benchMark(json = jfile) tdSql.execute(f"use {self.db}") tdSql.execute("select database();") - # set insert data information + # come from query_basic.json self.childtable_count = 6 self.insert_rows = 100000 self.timestamp_step = 30000 + self.start_timestamp = 1700000000000 + + + def genTime(self, preCnt, cnt): + start = self.start_timestamp + preCnt * self.timestamp_step + end = start + self.timestamp_step * cnt + return (start, end) + + + def doWindowQuery(self): + pre = f"select count(ts) from {self.stb} " + # case1 operator "in" "and" is same + cnt = 6000 + s,e = self.genTime(12000, cnt) + sql1 = f"{pre} where ts between {s} and {e} " + sql2 = f"{pre} where ts >= {s} and ts <={e} " + expectCnt = (cnt + 1) * self.childtable_count + tdSql.checkFirstValue(sql1, expectCnt) + tdSql.checkFirstValue(sql2, expectCnt) + + # case2 no overloap "or" left + cnt1 = 120 + s1, e1 = self.genTime(4000, cnt1) + cnt2 = 3000 + s2, e2 = self.genTime(10000, cnt2) + sql = f"{pre} where (ts >= {s1} and ts < {e1}) or (ts >= {s2} and ts < {e2})" + expectCnt = (cnt1 + cnt2) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case3 overloap "or" right + cnt1 = 300 + s1, e1 = self.genTime(17000, cnt1) + cnt2 = 8000 + s2, e2 = self.genTime(70000, cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})" + expectCnt = (cnt1 + cnt2) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case4 overloap "or" + cnt1 = 1000 + s1, e1 = self.genTime(9000, cnt1) + cnt2 = 1000 + s2, e2 = self.genTime(9000 + 500 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})" + expectCnt = (cnt1 + 500) * self.childtable_count # expect=1500 + tdSql.checkFirstValue(sql, expectCnt) + + # case5 overloap "or" boundary hollow->solid + cnt1 = 3000 + s1, e1 = self.genTime(45000, cnt1) + cnt2 = 2000 + s2, e2 = self.genTime(45000 + cnt1 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) or (ts > {s2} and ts <= {e2})" + expectCnt = (cnt1+cnt2) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case6 overloap "or" boundary solid->solid + cnt1 = 300 + s1, e1 = self.genTime(55000, cnt1) + cnt2 = 500 + s2, e2 = self.genTime(55000 + cnt1 , cnt2) + sql = f"{pre} where (ts >= {s1} and ts <= {e1}) or (ts >= {s2} and ts <= {e2})" + expectCnt = (cnt1+cnt2+1) * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case7 overloap "and" + cnt1 = 1000 + s1, e1 = self.genTime(40000, cnt1) + cnt2 = 1000 + s2, e2 = self.genTime(40000 + 500 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts > {s2} and ts <= {e2})" + expectCnt = cnt1/2 * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case8 overloap "and" boundary hollow->solid solid->hollow + cnt1 = 3000 + s1, e1 = self.genTime(45000, cnt1) + cnt2 = 2000 + s2, e2 = self.genTime(45000 + cnt1 , cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts < {e2})" + expectCnt = 1 * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + # case9 no overloap "and" + cnt1 = 6000 + s1, e1 = self.genTime(20000, cnt1) + cnt2 = 300 + s2, e2 = self.genTime(70000, cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts <= {e2})" + expectCnt = 0 + tdSql.checkFirstValue(sql, expectCnt) + + # case10 cnt1 contain cnt2 and + cnt1 = 5000 + s1, e1 = self.genTime(25000, cnt1) + cnt2 = 400 + s2, e2 = self.genTime(28000, cnt2) + sql = f"{pre} where (ts > {s1} and ts <= {e1}) and (ts >= {s2} and ts < {e2})" + expectCnt = cnt2 * self.childtable_count + tdSql.checkFirstValue(sql, expectCnt) + + + def queryMax(self, colname): + sql = f"select max({colname}) from {self.stb}" + tdSql.query(sql) + return tdSql.getData(0, 0) + + + def checkMax(self): + # max for tsdbRetrieveDatablockSMA2 coverage + colname = "ui" + max = self.queryMax(colname) + + # insert over max + sql = f"insert into d0(ts, {colname}) values" + for i in range(1, 5): + sql += f" (now + {i}s, {max+i})" + tdSql.execute(sql) + self.flushDb() + + expectMax = max + 4 + for i in range(1, 5): + realMax = self.queryMax(colname) + if realMax != expectMax: + tdLog.exit(f"Max value not expect. expect:{expectMax} real:{realMax}") + sql = f"delete from d0 where ui={expectMax}" + tdSql.execute(sql) + expectMax -= 1 + + self.checkInsertCorrect() def doQuery(self): tdLog.info(f"do query.") - + self.doWindowQuery() + + # max + self.checkMax() + # __group_key sql = f"select count(*),_group_key(uti),uti from {self.stb} partition by uti" tdSql.query(sql) @@ -89,6 +224,7 @@ class TDTestCase(TBase): if int(reals[k]) != v: tdLog.exit(f"distribute {k} expect: {v} real: {reals[k]}") + # run def run(self): tdLog.debug(f"start to excute {__file__}") From e1b0d4a555b383232241a39c83d728886d0ac60d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:26:34 +0800 Subject: [PATCH 3/7] fix: tweak delete max value --- tests/army/community/query/query_basic.py | 9 ++++++++- tests/army/frame/sql.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index eb332b8c71..575655c2b1 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -172,7 +172,14 @@ class TDTestCase(TBase): realMax = self.queryMax(colname) if realMax != expectMax: tdLog.exit(f"Max value not expect. expect:{expectMax} real:{realMax}") - sql = f"delete from d0 where ui={expectMax}" + + # query ts list + sql = f"select ts from d0 where ui={expectMax}" + tdSql.query(sql) + tss = tdSql.getColData(0) + strts = ",".join(tss) + # delete + sql = f"delete from d0 where ts in ({strts})" tdSql.execute(sql) expectMax -= 1 diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 8b7538321d..83f1243167 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -223,6 +223,12 @@ class TDSql: def getData(self, row, col): self.checkRowCol(row, col) return self.res[row][col] + + def getColData(self, col): + colDatas = [] + for i in range(self.queryRows): + colDatas.append(self.res[i][col]) + return colDatas def getResult(self, sql): self.sql = sql From 684bc221300e35cf352c8fbd9617d0ac9ccea75c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:26:41 +0800 Subject: [PATCH 4/7] fix: tweak delete max value --- tests/army/community/query/query_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 575655c2b1..740d14e11b 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -177,7 +177,7 @@ class TDTestCase(TBase): sql = f"select ts from d0 where ui={expectMax}" tdSql.query(sql) tss = tdSql.getColData(0) - strts = ",".join(tss) + strts = ",".join(map(str,tss)) # delete sql = f"delete from d0 where ts in ({strts})" tdSql.execute(sql) From 6cd689ea53714bf8035eb32bb66f61badd6c428c Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:43:00 +0800 Subject: [PATCH 5/7] fix: delete max ok --- tests/army/community/query/query_basic.py | 8 ++++---- tests/army/frame/caseBase.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 740d14e11b..211d3e6ec3 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -177,10 +177,10 @@ class TDTestCase(TBase): sql = f"select ts from d0 where ui={expectMax}" tdSql.query(sql) tss = tdSql.getColData(0) - strts = ",".join(map(str,tss)) - # delete - sql = f"delete from d0 where ts in ({strts})" - tdSql.execute(sql) + for ts in tss: + # delete + sql = f"delete from d0 where ts = '{ts}'" + tdSql.execute(sql) expectMax -= 1 self.checkInsertCorrect() diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 5715867faf..de258e6267 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -292,3 +292,15 @@ class TBase: if len(lists) == 0: tdLog.exit(f"list is empty {tips}") + +# +# str util +# + # covert list to sql format string + def listSql(self, lists, sepa = ","): + strs = "" + for ls in lists: + if strs != "": + strs += sepa + strs += f"'{ls}'" + return strs \ No newline at end of file From 68b2a06a02289bea1d9e4e8f4d93530eac3627a0 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 14:52:40 +0800 Subject: [PATCH 6/7] feat: add cquery_basic.json write again --- tests/army/community/query/cquery_basic.json | 61 ++++++++++++++++++++ tests/army/community/query/query_basic.py | 6 +- 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/army/community/query/cquery_basic.json diff --git a/tests/army/community/query/cquery_basic.json b/tests/army/community/query/cquery_basic.json new file mode 100644 index 0000000000..5a57d59d93 --- /dev/null +++ b/tests/army/community/query/cquery_basic.json @@ -0,0 +1,61 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 4000, + "prepared_rand": 10000, + "thread_count": 3, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "no", + "vgroups": 3, + "replica": 3, + "duration":"3d", + "wal_retention_period": 1, + "wal_retention_size": 1, + "stt_trigger": 1 + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "yes", + "childtable_count": 6, + "insert_rows": 50000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 60000, + "start_timestamp":1700000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti"}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 8}, + { "type": "nchar", "name": "nch", "len": 16} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/community/query/query_basic.py b/tests/army/community/query/query_basic.py index 211d3e6ec3..912974d8ab 100644 --- a/tests/army/community/query/query_basic.py +++ b/tests/army/community/query/query_basic.py @@ -49,6 +49,11 @@ class TDTestCase(TBase): self.timestamp_step = 30000 self.start_timestamp = 1700000000000 + # write again disorder + self.flushDb() + jfile = etool.curFile(__file__, "cquery_basic.json") + etool.benchMark(json = jfile) + def genTime(self, preCnt, cnt): start = self.start_timestamp + preCnt * self.timestamp_step @@ -238,7 +243,6 @@ class TDTestCase(TBase): # insert data self.insertData() - self.flushDb() # check insert data correct self.checkInsertCorrect() From c2772994e43d7fa08a54c95f6ece2ee5040636e4 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 20 Jan 2024 15:00:14 +0800 Subject: [PATCH 7/7] fix: add order by desc coverage --- tests/army/frame/caseBase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index de258e6267..f562ea1086 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -136,7 +136,7 @@ class TBase: tdSql.checkAgg(sql, self.childtable_count) # check step - sql = f"select * from (select diff(ts) as dif from {self.stb} partition by tbname) where dif != {self.timestamp_step}" + sql = f"select * from (select diff(ts) as dif from {self.stb} partition by tbname order by ts desc) where dif != {self.timestamp_step}" tdSql.query(sql) tdSql.checkRows(0)