From fb61efa8c474cb4da5ef6797cef962249608b750 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 14 Jun 2022 15:15:50 +0800 Subject: [PATCH 01/20] add case for distribute aggregate plan of max function --- tests/system-test/2-query/max.py | 144 ++++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index 5342c7d449..e519eeb207 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -5,6 +5,10 @@ import numpy as np class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) @@ -14,7 +18,141 @@ class TDTestCase: def prepare_data(self): - pass + pass + + def check_max_functions(self, tbname , col_name): + + max_sql = f"select max({col_name}) from {tbname};" + + same_sql = f"select {col_name} from {tbname} order by {col_name} desc limit 1" + + tdSql.query(max_sql) + max_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if max_result !=same_result: + tdLog.exit(" max function work not as expected, sql : %s "% max_sql) + else: + tdLog.info(" max function work as expected, sql : %s "% max_sql) + + + def support_distributed_aggregate(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 days 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + self.check_max_functions(tablename,colname) + + # max function with basic filter + print(vnode_tables) + + + def run(self): tdSql.prepare() @@ -197,6 +335,10 @@ class TDTestCase: tdSql.checkData(0, 0, np.max(floatData)) tdSql.query("select max(col1) from stb_1 where col2<=5") tdSql.checkData(0,0,5) + + self.support_distributed_aggregate() + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From c3c8fdd4b894dd10dd0da8d4cf1a48f00e5db67a Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Tue, 14 Jun 2022 16:22:56 +0800 Subject: [PATCH 02/20] update max.py --- tests/system-test/2-query/max.py | 239 ++++++++----------------------- 1 file changed, 57 insertions(+), 182 deletions(-) diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index e519eeb207..19dd55a0e6 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -15,10 +15,62 @@ class TDTestCase: self.rowNum = 10 self.ts = 1537146000000 + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + def max_check_stb_and_tb_base(self): + tdSql.prepare() + intData = [] + floatData = [] + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, + col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table stb_1 using stb tags('beijing')") + for i in range(self.rowNum): + tdSql.execute(f"insert into stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in ['ts','col11','col12','col13']: + for j in ['db.stb','stb','db.stb_1','stb_1']: + tdSql.error(f'select max({i} from {j} )') - def prepare_data(self): + for i in range(1,11): + for j in ['db.stb','stb','db.stb_1','stb_1']: + tdSql.query(f"select max(col{i}) from {j}") + if i<9: + tdSql.checkData(0, 0, np.max(intData)) + elif i>=9: + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from stb_1 where col2<=5") + tdSql.checkData(0,0,5) + tdSql.query("select max(col1) from stb where col2<=5") + tdSql.checkData(0,0,5) + tdSql.execute('drop database db') - pass + def max_check_ntb_base(self): + tdSql.prepare() + intData = [] + floatData = [] + tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, + col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20))''') + for i in range(self.rowNum): + tdSql.execute(f"insert into ntb values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + for i in ['ts','col11','col12','col13']: + for j in ['db.ntb','ntb']: + tdSql.error(f'select max({i} from {j} )') + for i in range(1,11): + for j in ['db.ntb','ntb']: + tdSql.query(f"select max(col{i}) from {j}") + if i<9: + tdSql.checkData(0, 0, np.max(intData)) + elif i>=9: + tdSql.checkData(0, 0, np.max(floatData)) + tdSql.query("select max(col1) from ntb where col2<=5") + tdSql.checkData(0,0,5) + tdSql.execute('drop database db') + def check_max_functions(self, tbname , col_name): @@ -153,188 +205,11 @@ class TDTestCase: - def run(self): - tdSql.prepare() - - intData = [] - floatData = [] - - tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') - tdSql.execute("create table stb_1 using stb tags('beijing')") - tdSql.execute('''create table ntb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') - for i in range(self.rowNum): - tdSql.execute("insert into ntb values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - intData.append(i + 1) - floatData.append(i + 0.1) - for i in range(self.rowNum): - tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - intData.append(i + 1) - floatData.append(i + 0.1) + def run(self): # max verifacation - tdSql.error("select max(ts) from stb_1") - tdSql.error("select max(ts) from db.stb_1") - tdSql.error("select max(col7) from stb_1") - tdSql.error("select max(col7) from db.stb_1") - tdSql.error("select max(col8) from stb_1") - tdSql.error("select max(col8) from db.stb_1") - tdSql.error("select max(col9) from stb_1") - tdSql.error("select max(col9) from db.stb_1") - - tdSql.query("select max(col1) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col1) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col2) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col2) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col3) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col3) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col4) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col4) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col11) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col11) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col12) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col12) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col13) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col13) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col14) from stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col14) from db.stb_1") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col5) from stb_1") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col5) from db.stb_1") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col6) from stb_1") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col6) from db.stb_1") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col1) from stb_1 where col2<=5") - tdSql.checkData(0,0,5) - - - - tdSql.error("select max(ts) from stb") - tdSql.error("select max(ts) from db.stb") - tdSql.error("select max(col7) from stb") - tdSql.error("select max(col7) from db.stb") - tdSql.error("select max(col8) from stb") - tdSql.error("select max(col8) from db.stb") - tdSql.error("select max(col9) from stb") - tdSql.error("select max(col9) from db.stb") - - tdSql.query("select max(col1) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col1) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col2) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col2) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col3) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col3) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col4) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col4) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col11) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col11) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col12) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col12) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col13) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col13) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col14) from stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col14) from db.stb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col5) from stb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col5) from db.stb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col6) from stb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col6) from db.stb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col1) from stb where col2<=5") - tdSql.checkData(0,0,5) - - - - tdSql.error("select max(ts) from ntb") - tdSql.error("select max(ts) from db.ntb") - tdSql.error("select max(col7) from ntb") - tdSql.error("select max(col7) from db.ntb") - tdSql.error("select max(col8) from ntb") - tdSql.error("select max(col8) from db.ntb") - tdSql.error("select max(col9) from ntb") - tdSql.error("select max(col9) from db.ntb") - - tdSql.query("select max(col1) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col1) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col2) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col2) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col3) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col3) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col4) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col4) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col11) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col11) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col12) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col12) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col13) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col13) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col14) from ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col14) from db.ntb") - tdSql.checkData(0, 0, np.max(intData)) - tdSql.query("select max(col5) from ntb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col5) from db.ntb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col6) from ntb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col6) from db.ntb") - tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query("select max(col1) from stb_1 where col2<=5") - tdSql.checkData(0,0,5) + self.max_check_stb_and_tb_base() + self.max_check_ntb_base() self.support_distributed_aggregate() From 37eeed02a7d4d41647538f20bf4a47a465c7c168 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 15 Jun 2022 15:14:25 +0800 Subject: [PATCH 03/20] add case for agg max functions --- .../system-test/2-query/distribute_agg_max.py | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 tests/system-test/2-query/distribute_agg_max.py diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py new file mode 100644 index 0000000000..9f041d3ebb --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -0,0 +1,232 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + + def check_max_functions(self, tbname , col_name): + + max_sql = f"select max({col_name}) from {tbname};" + + same_sql = f"select {col_name} from {tbname} order by {col_name} desc limit 1" + + tdSql.query(max_sql) + max_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if max_result !=same_result: + tdLog.exit(" max function work not as expected, sql : %s "% max_sql) + else: + tdLog.info(" max function work as expected, sql : %s "% max_sql) + + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 days 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def check_max_distribute_diff_vnode(self,col_name): + + vgroup_ids = [] + for k ,v in self.vnode_disbutes.items(): + if len(v)>=2: + vgroup_ids.append(k) + + distribute_tbnames = [] + + for vgroup_id in vgroup_ids: + vnode_tables = self.vnode_disbutes[vgroup_id] + distribute_tbnames.append(random.sample(vnode_tables,1)[0]) + tbname_ins = "" + for tbname in distribute_tbnames: + tbname_ins += "'%s' ,"%tbname + + tbname_filters = tbname_ins[:-1] + + max_sql = f"select max({col_name}) from stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) order by {col_name} desc limit 1" + + tdSql.query(max_sql) + max_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if max_result !=same_result: + tdLog.exit(" max function work not as expected, sql : %s "% max_sql) + else: + tdLog.info(" max function work as expected, sql : %s "% max_sql) + + def check_max_status(self): + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + self.check_max_functions(tablename,colname) + + # check max function for different vnode + + for colname in colnames: + if colname.startswith("c"): + self.check_max_distribute_diff_vnode(colname) + else: + # self.check_max_distribute_diff_vnode(colname) # bug for tag + pass + + + def distribute_agg_query(self): + # basic filter + tdSql.query("select max(c1) from stb1 where c1 is null") + tdSql.checkRows(0) + + tdSql.query("select max(c1) from stb1 where t1=1") + tdSql.checkData(0,0,10) + + tdSql.query("select max(c1+c2) from stb1 where c1 =1 ") + tdSql.checkData(0,0,11112.000000000) + + tdSql.query("select max(c1) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,10) + + tdSql.query("select max(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select max(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + tdSql.query("select max(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.check_max_status() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 8c6c476762d33c524b49c28890e936aa9aa295f9 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 16 Jun 2022 10:44:24 +0800 Subject: [PATCH 04/20] add case for distribute agg max function --- .../system-test/2-query/distribute_agg_max.py | 65 ++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index 9f041d3ebb..6897d7dc4f 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -212,9 +212,70 @@ class TDTestCase: tdSql.query("select max(c1) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - tdSql.query("select max(c1) from stb1 where t1> 4 partition by tbname") - tdSql.checkRows(15) + # union all + tdSql.query("select max(c1) from stb1 union all select max(c1) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,28) + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select max(tb1.c1), tb2.c2 from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,9) + tdSql.checkData(0,0,9.00000) + + # group by + tdSql.execute(" use testdb ") + tdSql.query(" select max(c1),c1 from stb1 group by t1 ") + tdSql.checkRows(20) + tdSql.query(" select max(c1),c1 from stb1 group by c1 ") + tdSql.checkRows(30) + tdSql.query(" select max(c1),c2 from stb1 group by c2 ") + tdSql.checkRows(31) + + # partition by tbname or partition by tag + tdSql.query("select max(c1),tbname from stb1 partition by tbname") + query_data = tdSql.queryResult + + for row in query_data: + tbname = row[1] + tdSql.query(" select max(c1) from %s "%tbname) + tdSql.checkData(0,0,row[0]) + + tdSql.query("select max(c1),tbname from stb1 partition by t1") + query_data = tdSql.queryResult + + for row in query_data: + tbname = row[1] + tdSql.query(" select max(c1) from %s "%tbname) + tdSql.checkData(0,0,row[0]) + + # nest query for support max + tdSql.query("select abs(c2+2)+1 from (select max(c1) c2 from stb1)") + tdSql.checkData(0,0,31.000000000) + tdSql.query("select max(c1+2)+1 as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,31.000000000) + tdSql.query("select max(a+2)+1 as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,31.000000000) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) def run(self): From c39f0a1f4720409cd045deca916756edad76fffc Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 16 Jun 2022 11:32:21 +0800 Subject: [PATCH 05/20] update case for duration for create db --- tests/system-test/2-query/distribute_agg_max.py | 2 +- tests/system-test/2-query/max.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index 6897d7dc4f..ae0ab5aafa 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -39,7 +39,7 @@ class TDTestCase: def prepare_datas_of_distribute(self): # prepate datas for 20 tables distributed at different vgroups - tdSql.execute("create database if not exists testdb keep 3650 days 1000 vgroups 5") + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") tdSql.execute( '''create table stb1 diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index 19dd55a0e6..0cb470088b 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -93,7 +93,7 @@ class TDTestCase: def support_distributed_aggregate(self): # prepate datas for 20 tables distributed at different vgroups - tdSql.execute("create database if not exists testdb keep 3650 days 1000 vgroups 5") + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") tdSql.execute( '''create table stb1 From 41bf8e850ed0d8181cc5e820c7ee065cb8a2b65c Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 16 Jun 2022 14:28:36 +0800 Subject: [PATCH 06/20] add test case for distribute plan about count function --- .../2-query/distribute_agg_count.py | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 tests/system-test/2-query/distribute_agg_count.py diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py new file mode 100644 index 0000000000..2ac9c86df0 --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -0,0 +1,296 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + + def check_count_functions(self, tbname , col_name): + + max_sql = f"select count({col_name}) from {tbname};" + + same_sql = f"select sum(c) from (select {col_name} ,1 as c from {tbname} where {col_name} is not null) " + + tdSql.query(max_sql) + max_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if max_result !=same_result: + tdLog.exit(" count function work not as expected, sql : %s "% max_sql) + else: + tdLog.info(" count function work as expected, sql : %s "% max_sql) + + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def check_count_distribute_diff_vnode(self,col_name): + + vgroup_ids = [] + for k ,v in self.vnode_disbutes.items(): + if len(v)>=2: + vgroup_ids.append(k) + + distribute_tbnames = [] + + for vgroup_id in vgroup_ids: + vnode_tables = self.vnode_disbutes[vgroup_id] + distribute_tbnames.append(random.sample(vnode_tables,1)[0]) + tbname_ins = "" + for tbname in distribute_tbnames: + tbname_ins += "'%s' ,"%tbname + + tbname_filters = tbname_ins[:-1] + + max_sql = f"select count({col_name}) from stb1 where tbname in ({tbname_filters});" + + same_sql = f"select sum(c) from (select {col_name} ,1 as c from stb1 where tbname in ({tbname_filters}) and {col_name} is not null) " + + tdSql.query(max_sql) + max_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if max_result !=same_result: + tdLog.exit(" count function work not as expected, sql : %s "% max_sql) + else: + tdLog.info(" count function work as expected, sql : %s "% max_sql) + + def check_count_status(self): + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + self.check_count_functions(tablename,colname) + + # check max function for different vnode + + for colname in colnames: + if colname.startswith("c"): + self.check_count_distribute_diff_vnode(colname) + else: + # self.check_count_distribute_diff_vnode(colname) # bug for tag + pass + + + def distribute_agg_query(self): + # basic filter + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,184) + + tdSql.query("select count(c1) from stb1 where t1=1") + tdSql.checkData(0,0,9) + + tdSql.query("select count(c1+c2) from stb1 where c1 =1 ") + tdSql.checkData(0,0,2) + + tdSql.query("select count(c1) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,9) + + tdSql.query("select count(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select count(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + # union all + tdSql.query("select count(c1) from stb1 union all select count(c1) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,184) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select count(tb1.c1), count(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,10) + tdSql.checkData(0,1,10) + + # group by + tdSql.execute(" use testdb ") + + tdSql.query(" select count(*) from stb1 ") + tdSql.checkData(0,0,187) + tdSql.query(" select count(*) from stb1 group by t1 ") + tdSql.checkRows(20) + tdSql.query(" select count(*) from stb1 group by c1 ") + tdSql.checkRows(30) + tdSql.query(" select count(*) from stb1 group by c2 ") + tdSql.checkRows(31) + + # partition by tbname or partition by tag + tdSql.query("select max(c1),tbname from stb1 partition by tbname") + query_data = tdSql.queryResult + + for row in query_data: + tbname = row[1] + tdSql.query(" select max(c1) from %s "%tbname) + tdSql.checkData(0,0,row[0]) + + tdSql.query("select max(c1),tbname from stb1 partition by t1") + query_data = tdSql.queryResult + + for row in query_data: + tbname = row[1] + tdSql.query(" select max(c1) from %s "%tbname) + tdSql.checkData(0,0,row[0]) + + # nest query for support max + tdSql.query("select abs(c2+2)+1 from (select count(c1) c2 from stb1)") + tdSql.checkData(0,0,187.000000000) + tdSql.query("select count(c1+2) as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,184) + tdSql.query("select count(a+2) as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,184) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.check_count_status() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From fe80b57fb8a0e29601e6f802da55a1a2d14fd62f Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 16 Jun 2022 14:29:30 +0800 Subject: [PATCH 07/20] update case --- tests/system-test/fulltest.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 552ed26f34..e3f8a71a5a 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -98,6 +98,9 @@ python3 ./test.py -f 2-query/stateduration.py python3 ./test.py -f 2-query/function_stateduration.py python3 ./test.py -f 2-query/statecount.py python3 ./test.py -f 2-query/tail.py +python3 ./test.py -f 2-query/distribute_agg_count.py +python3 ./test.py -f 2-query/distribute_agg_max.py + python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py From bef99f1c5ebeabd143f4f9abbae878d46e0261bc Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 16 Jun 2022 15:12:40 +0800 Subject: [PATCH 08/20] add case for distribute plan of min function --- .../system-test/2-query/distribute_agg_min.py | 294 ++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 tests/system-test/2-query/distribute_agg_min.py diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py new file mode 100644 index 0000000000..d560e962e2 --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -0,0 +1,294 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + + def check_min_functions(self, tbname , col_name): + + min_sql = f"select min({col_name}) from {tbname};" + + same_sql = f"select {col_name} from {tbname} where {col_name} is not null order by {col_name} asc limit 1" + + tdSql.query(min_sql) + min_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if min_result !=same_result: + tdLog.exit(" min function work not as expected, sql : %s "% min_sql) + else: + tdLog.info(" min function work as expected, sql : %s "% min_sql) + + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def check_min_distribute_diff_vnode(self,col_name): + + vgroup_ids = [] + for k ,v in self.vnode_disbutes.items(): + if len(v)>=2: + vgroup_ids.append(k) + + distribute_tbnames = [] + + for vgroup_id in vgroup_ids: + vnode_tables = self.vnode_disbutes[vgroup_id] + distribute_tbnames.append(random.sample(vnode_tables,1)[0]) + tbname_ins = "" + for tbname in distribute_tbnames: + tbname_ins += "'%s' ,"%tbname + + tbname_filters = tbname_ins[:-1] + + min_sql = f"select min({col_name}) from stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null order by {col_name} asc limit 1" + + tdSql.query(min_sql) + min_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if min_result !=same_result: + tdLog.exit(" min function work not as expected, sql : %s "% min_sql) + else: + tdLog.info(" min function work as expected, sql : %s "% min_sql) + + def check_min_status(self): + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + self.check_min_functions(tablename,colname) + + # check max function for different vnode + + for colname in colnames: + if colname.startswith("c"): + self.check_min_distribute_diff_vnode(colname) + else: + # self.check_max_distribute_diff_vnode(colname) # bug for tag + pass + + + def distribute_agg_query(self): + # basic filter + tdSql.query("select min(c1) from stb1 where c1 is null") + tdSql.checkRows(0) + + tdSql.query("select min(c1) from stb1 where t1=1") + tdSql.checkData(0,0,2) + + tdSql.query("select min(c1+c2) from stb1 where c1 =1 ") + tdSql.checkData(0,0,11112.000000000) + + tdSql.query("select min(c1) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,2) + + tdSql.query("select min(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select min(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + # union all + tdSql.query("select min(c1) from stb1 union all select min(c1) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,0) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select min(tb1.c1), tb2.c2 from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.checkData(0,0,0.00000) + + # group by + tdSql.execute(" use testdb ") + tdSql.query(" select min(c1),c1 from stb1 group by t1 ") + tdSql.checkRows(20) + tdSql.query(" select min(c1),c1 from stb1 group by c1 ") + tdSql.checkRows(30) + tdSql.query(" select min(c1),c2 from stb1 group by c2 ") + tdSql.checkRows(31) + + # partition by tbname or partition by tag + tdSql.query("select min(c1),tbname from stb1 partition by tbname") + query_data = tdSql.queryResult + + for row in query_data: + tbname = row[1] + tdSql.query(" select min(c1) from %s "%tbname) + tdSql.checkData(0,0,row[0]) + + tdSql.query("select min(c1),tbname from stb1 partition by t1") + query_data = tdSql.queryResult + + for row in query_data: + tbname = row[1] + tdSql.query(" select min(c1) from %s "%tbname) + tdSql.checkData(0,0,row[0]) + + # nest query for support max + tdSql.query("select abs(c2+2)+1 from (select min(c1) c2 from stb1)") + tdSql.checkData(0,0,3.000000000) + tdSql.query("select min(c1+2)+1 as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,3.000000000) + tdSql.query("select min(a+2)+1 as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,3.000000000) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3),min(c1) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) + tdSql.checkData(0,4,0) + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.check_min_status() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From f30bf1a07d9f2cb5c79f1ad8dc5e8febd03830b8 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 16 Jun 2022 15:13:08 +0800 Subject: [PATCH 09/20] add case for distribute plan of min function --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index e3f8a71a5a..19c3d93e60 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -100,6 +100,7 @@ python3 ./test.py -f 2-query/statecount.py python3 ./test.py -f 2-query/tail.py python3 ./test.py -f 2-query/distribute_agg_count.py python3 ./test.py -f 2-query/distribute_agg_max.py +python3 ./test.py -f 2-query/distribute_agg_min.py python3 ./test.py -f 6-cluster/5dnode1mnode.py From a0b4ae2732a166736eb0a5310819c03a81d16fcf Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 17 Jun 2022 10:54:17 +0800 Subject: [PATCH 10/20] add case for distribute plan about sum function --- .../system-test/2-query/distribute_agg_sum.py | 278 ++++++++++++++++++ tests/system-test/fulltest.sh | 1 + 2 files changed, 279 insertions(+) create mode 100644 tests/system-test/2-query/distribute_agg_sum.py diff --git a/tests/system-test/2-query/distribute_agg_sum.py b/tests/system-test/2-query/distribute_agg_sum.py new file mode 100644 index 0000000000..428a68cae2 --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_sum.py @@ -0,0 +1,278 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random ,os ,sys +import platform + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + + def check_sum_functions(self, tbname , col_name): + + sum_sql = f"select sum({col_name}) from {tbname};" + + same_sql = f"select {col_name} from {tbname} where {col_name} is not null " + + tdSql.query(same_sql) + pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') + pre_sum = np.sum(pre_data) + + tdSql.query(sum_sql) + tdSql.checkData(0,0,pre_sum) + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def check_sum_distribute_diff_vnode(self,col_name): + + vgroup_ids = [] + for k ,v in self.vnode_disbutes.items(): + if len(v)>=2: + vgroup_ids.append(k) + + distribute_tbnames = [] + + for vgroup_id in vgroup_ids: + vnode_tables = self.vnode_disbutes[vgroup_id] + distribute_tbnames.append(random.sample(vnode_tables,1)[0]) + tbname_ins = "" + for tbname in distribute_tbnames: + tbname_ins += "'%s' ,"%tbname + + tbname_filters = tbname_ins[:-1] + + sum_sql = f"select sum({col_name}) from stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null " + + tdSql.query(same_sql) + pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') + pre_sum = np.sum(pre_data) + + tdSql.query(sum_sql) + tdSql.checkData(0,0,pre_sum) + + def check_sum_status(self): + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + self.check_sum_functions(tablename,colname) + + # check max function for different vnode + + for colname in colnames: + if colname.startswith("c"): + self.check_sum_distribute_diff_vnode(colname) + else: + # self.check_count_distribute_diff_vnode(colname) # bug for tag + pass + + + def distribute_agg_query(self): + # basic filter + tdSql.query(" select sum(c1) from stb1 ") + tdSql.checkData(0,0,2592) + + tdSql.query(" select sum(a) from (select sum(c1) a from stb1 partition by tbname) ") + tdSql.checkData(0,0,2592) + + tdSql.query(" select sum(c1) from stb1 where t1=1") + tdSql.checkData(0,0,54) + + tdSql.query("select sum(c1+c2) from stb1 where c1 =1 ") + tdSql.checkData(0,0,22224.000000000) + + tdSql.query("select sum(c1) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,54) + + tdSql.query("select sum(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select sum(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + # union all + tdSql.query("select sum(c1) from stb1 union all select sum(c1) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,2592) + + tdSql.query("select sum(a) from (select sum(c1) a from stb1 union all select sum(c1) a from stb1)") + tdSql.checkRows(1) + tdSql.checkData(0,0,5184) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select sum(tb1.c1), sum(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,45) + tdSql.checkData(0,1,45.000000000) + + # group by + tdSql.execute(" use testdb ") + + # partition by tbname or partition by tag + tdSql.query("select sum(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + # nest query for support max + tdSql.query("select abs(c2+2)+1 from (select sum(c1) c2 from stb1)") + tdSql.checkData(0,0,2595.000000000) + tdSql.query("select sum(c1+2) as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,2960.000000000) + tdSql.query("select sum(a+2) as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,2960.000000000) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3),sum(c1+c2) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) + tdSql.checkData(0,4,28202310.000000000) + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.check_sum_status() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 38d8546bc2..3df89cdcde 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -101,6 +101,7 @@ python3 ./test.py -f 2-query/tail.py python3 ./test.py -f 2-query/distribute_agg_count.py python3 ./test.py -f 2-query/distribute_agg_max.py python3 ./test.py -f 2-query/distribute_agg_min.py +python3 ./test.py -f 2-query/distribute_agg_sum.py python3 ./test.py -f 6-cluster/5dnode1mnode.py From d9c8c60b0eedb0bab79f860564fd9f3526109086 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 17 Jun 2022 11:25:59 +0800 Subject: [PATCH 11/20] add case for distribute plan spread --- .../system-test/2-query/distribute_agg_min.py | 2 +- .../2-query/distribute_agg_spread.py | 281 ++++++++++++++++++ .../system-test/2-query/distribute_agg_sum.py | 2 +- tests/system-test/fulltest.sh | 2 +- 4 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 tests/system-test/2-query/distribute_agg_spread.py diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index d560e962e2..8a458c74df 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -188,7 +188,7 @@ class TDTestCase: if colname.startswith("c"): self.check_min_distribute_diff_vnode(colname) else: - # self.check_max_distribute_diff_vnode(colname) # bug for tag + # self.check_min_distribute_diff_vnode(colname) # bug for tag pass diff --git a/tests/system-test/2-query/distribute_agg_spread.py b/tests/system-test/2-query/distribute_agg_spread.py new file mode 100644 index 0000000000..926c859632 --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_spread.py @@ -0,0 +1,281 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + + def check_spread_functions(self, tbname , col_name): + + spread_sql = f"select spread({col_name}) from {tbname};" + + same_sql = f"select max({col_name})-min({col_name}) from {tbname}" + + tdSql.query(spread_sql) + spread_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if spread_result !=same_result: + tdLog.exit(" max function work not as expected, sql : %s "% spread_sql) + else: + tdLog.info(" max function work as expected, sql : %s "% spread_sql) + + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def check_spread_distribute_diff_vnode(self,col_name): + + vgroup_ids = [] + for k ,v in self.vnode_disbutes.items(): + if len(v)>=2: + vgroup_ids.append(k) + + distribute_tbnames = [] + + for vgroup_id in vgroup_ids: + vnode_tables = self.vnode_disbutes[vgroup_id] + distribute_tbnames.append(random.sample(vnode_tables,1)[0]) + tbname_ins = "" + for tbname in distribute_tbnames: + tbname_ins += "'%s' ,"%tbname + + tbname_filters = tbname_ins[:-1] + + spread_sql = f"select spread({col_name}) from stb1 where tbname in ({tbname_filters})" + + same_sql = f"select max({col_name}) - min({col_name}) from stb1 where tbname in ({tbname_filters})" + + tdSql.query(spread_sql) + spread_result = tdSql.queryResult + + tdSql.query(same_sql) + same_result = tdSql.queryResult + + if spread_result !=same_result: + tdLog.exit(" spread function work not as expected, sql : %s "% spread_sql) + else: + tdLog.info(" spread function work as expected, sql : %s "% spread_sql) + + def check_spread_status(self): + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + self.check_spread_functions(tablename,colname) + + # check max function for different vnode + + for colname in colnames: + if colname.startswith("c"): + self.check_spread_distribute_diff_vnode(colname) + else: + # self.check_spread_distribute_diff_vnode(colname) # bug for tag + pass + + + def distribute_agg_query(self): + # basic filter + tdSql.query("select spread(c1) from stb1 where c1 is null") + tdSql.checkRows(0) + + tdSql.query("select spread(c1) from stb1 where t1=1") + tdSql.checkData(0,0,8.000000000) + + tdSql.query("select spread(c1+c2) from stb1 where c1 =1 ") + tdSql.checkData(0,0,0.000000000) + + tdSql.query("select spread(c1) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,8.000000000) + + tdSql.query("select spread(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select spread(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + # union all + tdSql.query("select spread(c1) from stb1 union all select max(c1)-min(c1) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,28.000000000) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select spread(tb1.c1), spread(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,9.000000000) + tdSql.checkData(0,0,9.00000) + + # group by + tdSql.execute(" use testdb ") + tdSql.query(" select max(c1),c1 from stb1 group by t1 ") + tdSql.checkRows(20) + tdSql.query(" select max(c1),c1 from stb1 group by c1 ") + tdSql.checkRows(30) + tdSql.query(" select max(c1),c2 from stb1 group by c2 ") + tdSql.checkRows(31) + + # partition by tbname or partition by tag + tdSql.query("select spread(c1),tbname from stb1 partition by tbname") + query_data = tdSql.queryResult + + # nest query for support max + tdSql.query("select spread(c2+2)+1 from (select max(c1) c2 from stb1)") + tdSql.checkData(0,0,1.000000000) + tdSql.query("select spread(c1+2)+1 as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,29.000000000) + tdSql.query("select spread(a+2)+1 as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,29.000000000) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3),spread(c1) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) + tdSql.checkData(0,4,28.000000000) + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.check_spread_status() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/distribute_agg_sum.py b/tests/system-test/2-query/distribute_agg_sum.py index 428a68cae2..add4d75c61 100644 --- a/tests/system-test/2-query/distribute_agg_sum.py +++ b/tests/system-test/2-query/distribute_agg_sum.py @@ -184,7 +184,7 @@ class TDTestCase: if colname.startswith("c"): self.check_sum_distribute_diff_vnode(colname) else: - # self.check_count_distribute_diff_vnode(colname) # bug for tag + # self.check_sum_distribute_diff_vnode(colname) # bug for tag pass diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 3df89cdcde..f64330d346 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -102,7 +102,7 @@ python3 ./test.py -f 2-query/distribute_agg_count.py python3 ./test.py -f 2-query/distribute_agg_max.py python3 ./test.py -f 2-query/distribute_agg_min.py python3 ./test.py -f 2-query/distribute_agg_sum.py - +python3 ./test.py -f 2-query/distribute_agg_spread.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py From df7887c63ee90f71a3821487954f71c64b417bb3 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 17 Jun 2022 15:35:18 +0800 Subject: [PATCH 12/20] add test case for distribute plan for apercentile --- .../2-query/distribute_agg_apercentile.py | 198 ++++++++++++++++++ tests/system-test/fulltest.sh | 1 + 2 files changed, 199 insertions(+) create mode 100644 tests/system-test/2-query/distribute_agg_apercentile.py diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py new file mode 100644 index 0000000000..d61532c945 --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -0,0 +1,198 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def distribute_agg_query(self): + # basic filter + tdSql.query("select apercentile(c1 , 20) from stb1 where c1 is null") + tdSql.checkRows(0) + + tdSql.query("select apercentile(c1 , 20) from stb1 where t1=1") + tdSql.checkData(0,0,2.800000000) + + tdSql.query("select apercentile(c1+c2 ,100) from stb1 where c1 =1 ") + tdSql.checkData(0,0,11112.000000000) + + tdSql.query("select apercentile(c1 ,10 ) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,2.000000000) + + tdSql.query("select apercentile(c1,20) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select apercentile(c1,20) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + # union all + tdSql.query("select apercentile(c1,20) from stb1 union all select apercentile(c1,10) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,7.389181281) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select apercentile(tb1.c1,100), apercentile(tb2.c2,100) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,9.000000000) + tdSql.checkData(0,0,9.000000000) + + # group by + tdSql.execute(" use testdb ") + tdSql.query(" select max(c1),c1 from stb1 group by t1 ") + tdSql.checkRows(20) + tdSql.query(" select max(c1),c1 from stb1 group by c1 ") + tdSql.checkRows(30) + tdSql.query(" select max(c1),c2 from stb1 group by c2 ") + tdSql.checkRows(31) + + # partition by tbname or partition by tag + tdSql.query("select apercentile(c1 ,10)from stb1 partition by tbname") + query_data = tdSql.queryResult + + # nest query for support max + tdSql.query("select apercentile(c2+2,10)+1 from (select max(c1) c2 from stb1)") + tdSql.checkData(0,0,31.000000000) + tdSql.query("select apercentile(c1+2,10)+1 as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,7.560701700) + tdSql.query("select apercentile(a+2,10)+1 as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,7.560701700) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3),spread(c1), apercentile(c1,10) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) + tdSql.checkData(0,4,28.000000000) + tdSql.checkData(0,5,4.560701700) + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index f64330d346..a4655bddbe 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -103,6 +103,7 @@ python3 ./test.py -f 2-query/distribute_agg_max.py python3 ./test.py -f 2-query/distribute_agg_min.py python3 ./test.py -f 2-query/distribute_agg_sum.py python3 ./test.py -f 2-query/distribute_agg_spread.py +python3 ./test.py -f 2-query/distribute_agg_apercentile.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py From b1d92f682a891622a09ece27eb4899e4ba4fbe75 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 17 Jun 2022 15:39:00 +0800 Subject: [PATCH 13/20] test: modify case --- tests/system-test/7-tmq/basic5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py index e44f327995..4ed3be967e 100644 --- a/tests/system-test/7-tmq/basic5.py +++ b/tests/system-test/7-tmq/basic5.py @@ -192,7 +192,7 @@ class TDTestCase: time.sleep(1) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 20 showMsg = 1 showRow = 1 @@ -208,7 +208,7 @@ class TDTestCase: os.system(shellCmd) # wait for data ready - prepareEnvThread.join() + # prepareEnvThread.join() tdLog.info("insert process end, and start to check consume result") while 1: From 5efb5deba9788a0b30cebd820950fa01121bb56b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 17 Jun 2022 16:11:58 +0800 Subject: [PATCH 14/20] update case --- tests/system-test/2-query/distribute_agg_apercentile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index d61532c945..fd1455ce16 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -130,7 +130,7 @@ class TDTestCase: tdSql.checkRows(15) # union all - tdSql.query("select apercentile(c1,20) from stb1 union all select apercentile(c1,10) from stb1 ") + tdSql.query("select apercentile(c1,20) from stb1 union all select apercentile(c1,20) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,7.389181281) From 9eff85b6807d563dd9898672c2633131a7eabee5 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 17 Jun 2022 17:36:37 +0800 Subject: [PATCH 15/20] update case for spread --- tests/system-test/2-query/distribute_agg_spread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/distribute_agg_spread.py b/tests/system-test/2-query/distribute_agg_spread.py index 926c859632..94f1a61d77 100644 --- a/tests/system-test/2-query/distribute_agg_spread.py +++ b/tests/system-test/2-query/distribute_agg_spread.py @@ -246,7 +246,7 @@ class TDTestCase: tdSql.checkRows(31) # partition by tbname or partition by tag - tdSql.query("select spread(c1),tbname from stb1 partition by tbname") + tdSql.query("select spread(c1) from stb1 partition by tbname") query_data = tdSql.queryResult # nest query for support max From 6c7dce7363b631ddcb719766c3690bd7548e471a Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 17 Jun 2022 18:06:08 +0800 Subject: [PATCH 16/20] fix: merge data in memory and file --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index fe89321ae9..dd278a7953 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -111,7 +111,7 @@ int32_t tsdbBegin(STsdb *pTsdb) { int32_t tsdbCommit(STsdb *pTsdb) { if (!pTsdb) return 0; - + int32_t code = 0; SCommitH commith = {0}; SDFileSet *pSet = NULL; @@ -495,7 +495,9 @@ static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { break; } - if (pIter && pIter->pTable && (!pIdx || (pIter->pTable->suid <= pIdx->suid || pIter->pTable->uid <= pIdx->uid))) { + if (pIter && pIter->pTable && + (!pIdx || ((pIter->pTable->suid < pIdx->suid) || + (pIter->pTable->suid == pIdx->suid && pIter->pTable->uid <= pIdx->uid)))) { if (tsdbCommitToTable(pCommith, mIter) < 0) { tsdbCloseCommitFile(pCommith, true); // revert the file change @@ -503,7 +505,7 @@ static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { return -1; } - if (pIdx && (pIter->pTable->uid == pIdx->uid)) { + if (pIdx && ((pIter->pTable->uid == pIdx->uid) && (pIter->pTable->suid == pIdx->suid))) { ++fIter; } ++mIter; @@ -517,7 +519,10 @@ static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); return -1; } + ++fIter; + } else { + ASSERT(0); } } From 715bb6d2e906500978809dca9198951f2d9e6aa7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 17 Jun 2022 18:08:41 +0800 Subject: [PATCH 17/20] other: code optimization --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index dd278a7953..284b6c9a4c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -497,7 +497,7 @@ static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { if (pIter && pIter->pTable && (!pIdx || ((pIter->pTable->suid < pIdx->suid) || - (pIter->pTable->suid == pIdx->suid && pIter->pTable->uid <= pIdx->uid)))) { + ((pIter->pTable->suid == pIdx->suid) && (pIter->pTable->uid <= pIdx->uid))))) { if (tsdbCommitToTable(pCommith, mIter) < 0) { tsdbCloseCommitFile(pCommith, true); // revert the file change From 22e62e5cd2e4833302761dc6a6283c2582dadaa3 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 17 Jun 2022 18:11:09 +0800 Subject: [PATCH 18/20] other: code optimization --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 284b6c9a4c..e98fd8ae1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -519,7 +519,6 @@ static int32_t tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); return -1; } - ++fIter; } else { ASSERT(0); From bda0327bbe41ffb058d0b6fd8c5aecaab49404f0 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 17 Jun 2022 20:24:46 +0800 Subject: [PATCH 19/20] test:split test case for timeout --- tests/system-test/7-tmq/subscribeDb0.py | 172 ------------ tests/system-test/7-tmq/subscribeDb1.py | 294 ++++++-------------- tests/system-test/7-tmq/subscribeDb2.py | 347 ++++++++++++++++++++++++ tests/system-test/7-tmq/subscribeDb3.py | 337 +++++++++++++++++++++++ 4 files changed, 760 insertions(+), 390 deletions(-) create mode 100644 tests/system-test/7-tmq/subscribeDb2.py create mode 100644 tests/system-test/7-tmq/subscribeDb3.py diff --git a/tests/system-test/7-tmq/subscribeDb0.py b/tests/system-test/7-tmq/subscribeDb0.py index c9f256ed74..4e8fb04517 100644 --- a/tests/system-test/7-tmq/subscribeDb0.py +++ b/tests/system-test/7-tmq/subscribeDb0.py @@ -322,176 +322,6 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 5 end ...... ") - def tmqCase6(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 6: Produce while one consumers to subscribe tow topic, Each contains one db") - tdLog.info("step 1: create database, stb, ctb and insert data") - # create and start thread - parameterDict = {'cfg': '', \ - 'dbName': 'db60', \ - 'vgroups': 4, \ - 'stbName': 'stb', \ - 'ctbNum': 10, \ - 'rowsPerTbl': 5000, \ - 'batchNum': 100, \ - 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 - parameterDict['cfg'] = cfgPath - - self.initConsumerTable() - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) - - prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) - prepareEnvThread.start() - - parameterDict2 = {'cfg': '', \ - 'dbName': 'db61', \ - 'vgroups': 4, \ - 'stbName': 'stb2', \ - 'ctbNum': 10, \ - 'rowsPerTbl': 5000, \ - 'batchNum': 100, \ - 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 - parameterDict['cfg'] = cfgPath - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) - - prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) - prepareEnvThread2.start() - - tdLog.info("create topics from db") - topicName1 = 'topic_db60' - topicName2 = 'topic_db61' - - tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) - tdSql.execute("create topic %s as database %s" %(topicName2, parameterDict2['dbName'])) - - consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] - topicList = topicName1 + ',' + topicName2 - ifcheckdata = 0 - ifManualCommit = 0 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:false,\ - auto.commit.interval.ms:6000,\ - auto.offset.reset:earliest' - self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - #consumerId = 1 - #self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - event.wait() - - tdLog.info("start consume processor") - pollDelay = 100 - showMsg = 1 - showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - - # wait for data ready - prepareEnvThread.join() - prepareEnvThread2.join() - - tdLog.info("insert process end, and start to check consume result") - expectRows = 1 - resultList = self.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - if totalConsumeRows != expectrowcnt: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - tdLog.exit("tmq consume rows error!") - - tdSql.query("drop topic %s"%topicName1) - tdSql.query("drop topic %s"%topicName2) - - tdLog.printNoPrefix("======== test case 6 end ...... ") - - def tmqCase7(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 7: Produce while two consumers to subscribe tow topic, Each contains one db") - tdLog.info("step 1: create database, stb, ctb and insert data") - # create and start thread - parameterDict = {'cfg': '', \ - 'dbName': 'db70', \ - 'vgroups': 4, \ - 'stbName': 'stb', \ - 'ctbNum': 10, \ - 'rowsPerTbl': 5000, \ - 'batchNum': 100, \ - 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 - parameterDict['cfg'] = cfgPath - - self.initConsumerTable() - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) - - prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) - prepareEnvThread.start() - - parameterDict2 = {'cfg': '', \ - 'dbName': 'db71', \ - 'vgroups': 4, \ - 'stbName': 'stb2', \ - 'ctbNum': 10, \ - 'rowsPerTbl': 5000, \ - 'batchNum': 100, \ - 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 - parameterDict['cfg'] = cfgPath - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) - - prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) - prepareEnvThread2.start() - - tdLog.info("create topics from db") - topicName1 = 'topic_db60' - topicName2 = 'topic_db61' - - tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) - tdSql.execute("create topic %s as database %s" %(topicName2, parameterDict2['dbName'])) - - consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] - topicList = topicName1 + ',' + topicName2 - ifcheckdata = 0 - ifManualCommit = 1 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:false,\ - auto.commit.interval.ms:6000,\ - auto.offset.reset:earliest' - self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - consumerId = 1 - self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - event.wait() - - tdLog.info("start consume processor") - pollDelay = 100 - showMsg = 1 - showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - - # wait for data ready - prepareEnvThread.join() - prepareEnvThread2.join() - - tdLog.info("insert process end, and start to check consume result") - expectRows = 2 - resultList = self.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - if totalConsumeRows != expectrowcnt: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - tdLog.exit("tmq consume rows error!") - - tdSql.query("drop topic %s"%topicName1) - tdSql.query("drop topic %s"%topicName2) - - tdLog.printNoPrefix("======== test case 7 end ...... ") - def run(self): tdSql.prepare() @@ -505,8 +335,6 @@ class TDTestCase: self.tmqCase4(cfgPath, buildPath) self.tmqCase5(cfgPath, buildPath) - self.tmqCase6(cfgPath, buildPath) - self.tmqCase7(cfgPath, buildPath) def stop(self): diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index c08c7d3dae..28a341f8f3 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -72,10 +72,10 @@ class TDTestCase: if tdSql.getRows() == expectRows: break else: - time.sleep(5) - + time.sleep(5) + for i in range(expectRows): - tdLog.info ("ts: %s, consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 0), tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) resultList.append(tdSql.getData(i , 3)) return resultList @@ -85,7 +85,7 @@ class TDTestCase: logFile = cfgPath + '/../log/valgrind-tmq.log' shellCmd = 'nohup valgrind --log-file=' + logFile shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' - + if (platform.system().lower() == 'windows'): shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) @@ -97,7 +97,7 @@ class TDTestCase: tdLog.info(shellCmd) os.system(shellCmd) - def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum): tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) @@ -151,8 +151,7 @@ class TDTestCase: parameterDict["dbName"],\ parameterDict["vgroups"],\ parameterDict["stbName"],\ - parameterDict["ctbNum"],\ - parameterDict["rowsPerTbl"]) + parameterDict["ctbNum"]) self.insert_data(tsql,\ parameterDict["dbName"],\ @@ -163,16 +162,16 @@ class TDTestCase: parameterDict["startTs"]) return - def tmqCase8(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 8: Produce while one consume to subscribe one db, inclue 1 stb") + def tmqCase6(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 6: Produce while one consumers to subscribe tow topic, Each contains one db") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread parameterDict = {'cfg': '', \ - 'dbName': 'db8', \ + 'dbName': 'db60', \ 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ + 'rowsPerTbl': 5000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -183,14 +182,32 @@ class TDTestCase: prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() - + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db61', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 5000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + tdLog.info("create topics from db") - topicName1 = 'topic_db1' + topicName1 = 'topic_db60' + topicName2 = 'topic_db61' + + tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) + tdSql.execute("create topic %s as database %s" %(topicName2, parameterDict2['dbName'])) - tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] / 2 - topicList = topicName1 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ',' + topicName2 ifcheckdata = 0 ifManualCommit = 0 keyList = 'group.id:cgrp1,\ @@ -199,6 +216,9 @@ class TDTestCase: auto.offset.reset:earliest' self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + #consumerId = 1 + #self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + event.wait() tdLog.info("start consume processor") @@ -208,7 +228,8 @@ class TDTestCase: self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) # wait for data ready - prepareEnvThread.join() + prepareEnvThread.join() + prepareEnvThread2.join() tdLog.info("insert process end, and start to check consume result") expectRows = 1 @@ -221,36 +242,21 @@ class TDTestCase: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") - - tdLog.info("again start consume processer") - self.initConsumerTable() - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] - self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - expectRows = 1 - resultList = self.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - if totalConsumeRows != expectrowcnt: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - tdLog.exit("tmq consume rows error!") - tdSql.query("drop topic %s"%topicName1) + tdSql.query("drop topic %s"%topicName2) - tdLog.printNoPrefix("======== test case 8 end ...... ") + tdLog.printNoPrefix("======== test case 6 end ...... ") - def tmqCase9(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 9: Produce while one consume to subscribe one db, inclue 1 stb") + def tmqCase7(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 7: Produce while two consumers to subscribe tow topic, Each contains one db") tdLog.info("step 1: create database, stb, ctb and insert data") # create and start thread parameterDict = {'cfg': '', \ - 'dbName': 'db9', \ + 'dbName': 'db70', \ 'vgroups': 4, \ 'stbName': 'stb', \ 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ + 'rowsPerTbl': 5000, \ 'batchNum': 100, \ 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 parameterDict['cfg'] = cfgPath @@ -261,14 +267,32 @@ class TDTestCase: prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) prepareEnvThread.start() - + + parameterDict2 = {'cfg': '', \ + 'dbName': 'db71', \ + 'vgroups': 4, \ + 'stbName': 'stb2', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 5000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict2['dbName'], parameterDict2['vgroups'])) + + prepareEnvThread2 = threading.Thread(target=self.prepareEnv, kwargs=parameterDict2) + prepareEnvThread2.start() + tdLog.info("create topics from db") - topicName1 = 'topic_db1' + topicName1 = 'topic_db60' + topicName2 = 'topic_db61' + + tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) + tdSql.execute("create topic %s as database %s" %(topicName2, parameterDict2['dbName'])) - tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] / 2 - topicList = topicName1 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + parameterDict2["rowsPerTbl"] * parameterDict2["ctbNum"] + topicList = topicName1 + ',' + topicName2 ifcheckdata = 0 ifManualCommit = 1 keyList = 'group.id:cgrp1,\ @@ -277,86 +301,7 @@ class TDTestCase: auto.offset.reset:earliest' self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - event.wait() - - tdLog.info("start consume processor") - pollDelay = 100 - showMsg = 1 - showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - - # wait for data ready - prepareEnvThread.join() - - tdLog.info("insert process end, and start to check consume result") - expectRows = 1 - resultList = self.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - tdSql.query("select count(*) from %s.%s" %(parameterDict['dbName'], parameterDict['stbName'])) - countOfStb = tdSql.getData(0,0) - print ("====total rows of stb: %d"%countOfStb) - - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) - if totalConsumeRows < expectrowcnt: - tdLog.exit("tmq consume rows error!") - - tdLog.info("again start consume processer") - self.initConsumerTable() - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] - self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - expectRows = 1 - resultList = self.selectConsumeResult(expectRows) - totalConsumeRows2 = 0 - for i in range(expectRows): - totalConsumeRows2 += resultList[i] - - tdLog.info("firstly act consume rows: %d"%(totalConsumeRows)) - tdLog.info("secondly act consume rows: %d, expect consume rows: %d"%(totalConsumeRows2, expectrowcnt)) - if totalConsumeRows + totalConsumeRows2 != expectrowcnt: - tdLog.exit("tmq consume rows error!") - - tdSql.query("drop topic %s"%topicName1) - - tdLog.printNoPrefix("======== test case 9 end ...... ") - - def tmqCase10(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 10: Produce while one consume to subscribe one db, inclue 1 stb") - tdLog.info("step 1: create database, stb, ctb and insert data") - # create and start thread - parameterDict = {'cfg': '', \ - 'dbName': 'db10', \ - 'vgroups': 4, \ - 'stbName': 'stb', \ - 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ - 'batchNum': 100, \ - 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 - parameterDict['cfg'] = cfgPath - - self.initConsumerTable() - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) - - prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) - prepareEnvThread.start() - - tdLog.info("create topics from db") - topicName1 = 'topic_db1' - - tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) - consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] - topicList = topicName1 - ifcheckdata = 0 - ifManualCommit = 1 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:false,\ - auto.commit.interval.ms:6000,\ - auto.offset.reset:earliest' + consumerId = 1 self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) event.wait() @@ -367,23 +312,12 @@ class TDTestCase: showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - time.sleep(2) - tdLog.info("pkill consume processor") - if (platform.system().lower() == 'windows'): - os.system("TASKKILL /F /IM tmq_sim.exe") - else: - os.system('pkill tmq_sim') - expectRows = 0 - resultList = self.selectConsumeResult(expectRows) - # wait for data ready prepareEnvThread.join() + prepareEnvThread2.join() + tdLog.info("insert process end, and start to check consume result") - - tdLog.info("again start consume processer") - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - - expectRows = 1 + expectRows = 2 resultList = self.selectConsumeResult(expectRows) totalConsumeRows = 0 for i in range(expectRows): @@ -393,85 +327,10 @@ class TDTestCase: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") - time.sleep(15) tdSql.query("drop topic %s"%topicName1) + tdSql.query("drop topic %s"%topicName2) - tdLog.printNoPrefix("======== test case 10 end ...... ") - - def tmqCase11(self, cfgPath, buildPath): - tdLog.printNoPrefix("======== test case 11: Produce while one consume to subscribe one db, inclue 1 stb") - tdLog.info("step 1: create database, stb, ctb and insert data") - # create and start thread - parameterDict = {'cfg': '', \ - 'dbName': 'db11', \ - 'vgroups': 4, \ - 'stbName': 'stb', \ - 'ctbNum': 10, \ - 'rowsPerTbl': 10000, \ - 'batchNum': 100, \ - 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 - parameterDict['cfg'] = cfgPath - - self.initConsumerTable() - - tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) - - prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) - prepareEnvThread.start() - - tdLog.info("create topics from db") - topicName1 = 'topic_db1' - - tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) - consumerId = 0 - expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] - topicList = topicName1 - ifcheckdata = 0 - ifManualCommit = 1 - keyList = 'group.id:cgrp1,\ - enable.auto.commit:true,\ - auto.commit.interval.ms:1000,\ - auto.offset.reset:earliest' - self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - - event.wait() - - tdLog.info("start consume processor") - pollDelay = 20 - showMsg = 1 - showRow = 1 - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - - time.sleep(6) - tdLog.info("pkill consume processor") - if (platform.system().lower() == 'windows'): - os.system("TASKKILL /F /IM tmq_sim.exe") - else: - os.system('pkill tmq_sim') - expectRows = 0 - resultList = self.selectConsumeResult(expectRows) - - # wait for data ready - prepareEnvThread.join() - tdLog.info("insert process end, and start to check consume result") - - tdLog.info("again start consume processer") - self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - - expectRows = 1 - resultList = self.selectConsumeResult(expectRows) - totalConsumeRows = 0 - for i in range(expectRows): - totalConsumeRows += resultList[i] - - if totalConsumeRows >= expectrowcnt or totalConsumeRows <= 0: - tdLog.info("act consume rows: %d, expect consume rows between %d and 0"%(totalConsumeRows, expectrowcnt)) - tdLog.exit("tmq consume rows error!") - - time.sleep(15) - tdSql.query("drop topic %s"%topicName1) - - tdLog.printNoPrefix("======== test case 11 end ...... ") + tdLog.printNoPrefix("======== test case 7 end ...... ") def run(self): tdSql.prepare() @@ -484,10 +343,9 @@ class TDTestCase: cfgPath = buildPath + "/../sim/psim/cfg" tdLog.info("cfgPath: %s" % cfgPath) - self.tmqCase8(cfgPath, buildPath) - self.tmqCase9(cfgPath, buildPath) - self.tmqCase10(cfgPath, buildPath) - self.tmqCase11(cfgPath, buildPath) + self.tmqCase6(cfgPath, buildPath) + self.tmqCase7(cfgPath, buildPath) + def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/subscribeDb2.py b/tests/system-test/7-tmq/subscribeDb2.py new file mode 100644 index 0000000000..af31e802b3 --- /dev/null +++ b/tests/system-test/7-tmq/subscribeDb2.py @@ -0,0 +1,347 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("ts: %s, consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 0), tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("use %s" %dbName) + tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + event.set() + tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + print ("input parameters:") + print (parameterDict) + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + self.create_tables(tsql,\ + parameterDict["dbName"],\ + parameterDict["vgroups"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"]) + + self.insert_data(tsql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"],\ + parameterDict["startTs"]) + return + + def tmqCase8(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 8: Produce while one consume to subscribe one db, inclue 1 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db8', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] / 2 + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + + tdLog.info("again start consume processer") + self.initConsumerTable() + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 8 end ...... ") + + def tmqCase9(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 9: Produce while one consume to subscribe one db, inclue 1 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db9', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] / 2 + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + # wait for data ready + prepareEnvThread.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdSql.query("select count(*) from %s.%s" %(parameterDict['dbName'], parameterDict['stbName'])) + countOfStb = tdSql.getData(0,0) + print ("====total rows of stb: %d"%countOfStb) + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + if totalConsumeRows < expectrowcnt: + tdLog.exit("tmq consume rows error!") + + tdLog.info("again start consume processer") + self.initConsumerTable() + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows2 = 0 + for i in range(expectRows): + totalConsumeRows2 += resultList[i] + + tdLog.info("firstly act consume rows: %d"%(totalConsumeRows)) + tdLog.info("secondly act consume rows: %d, expect consume rows: %d"%(totalConsumeRows2, expectrowcnt)) + if totalConsumeRows + totalConsumeRows2 != expectrowcnt: + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 9 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase8(cfgPath, buildPath) + self.tmqCase9(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py new file mode 100644 index 0000000000..6973f4c51f --- /dev/null +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -0,0 +1,337 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("ts: %s, consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 0), tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups)) + tsql.execute("use %s" %dbName) + tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + event.set() + tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s_%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + print ("input parameters:") + print (parameterDict) + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + self.create_tables(tsql,\ + parameterDict["dbName"],\ + parameterDict["vgroups"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"]) + + self.insert_data(tsql,\ + parameterDict["dbName"],\ + parameterDict["stbName"],\ + parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],\ + parameterDict["batchNum"],\ + parameterDict["startTs"]) + return + + def tmqCase10(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 10: Produce while one consume to subscribe one db, inclue 1 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db10', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:false,\ + auto.commit.interval.ms:6000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 100 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + time.sleep(2) + tdLog.info("pkill consume processor") + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM tmq_sim.exe") + else: + os.system('pkill tmq_sim') + expectRows = 0 + resultList = self.selectConsumeResult(expectRows) + + # wait for data ready + prepareEnvThread.join() + tdLog.info("insert process end, and start to check consume result") + + tdLog.info("again start consume processer") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + time.sleep(15) + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 10 end ...... ") + + def tmqCase11(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 11: Produce while one consume to subscribe one db, inclue 1 stb") + tdLog.info("step 1: create database, stb, ctb and insert data") + # create and start thread + parameterDict = {'cfg': '', \ + 'dbName': 'db11', \ + 'vgroups': 4, \ + 'stbName': 'stb', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.initConsumerTable() + + tdSql.execute("create database if not exists %s vgroups %d" %(parameterDict['dbName'], parameterDict['vgroups'])) + + prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict) + prepareEnvThread.start() + + tdLog.info("create topics from db") + topicName1 = 'topic_db1' + + tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] + topicList = topicName1 + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + event.wait() + + tdLog.info("start consume processor") + pollDelay = 20 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + time.sleep(6) + tdLog.info("pkill consume processor") + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM tmq_sim.exe") + else: + os.system('pkill tmq_sim') + expectRows = 0 + resultList = self.selectConsumeResult(expectRows) + + # wait for data ready + prepareEnvThread.join() + tdLog.info("insert process end, and start to check consume result") + + tdLog.info("again start consume processer") + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows >= expectrowcnt or totalConsumeRows <= 0: + tdLog.info("act consume rows: %d, expect consume rows between %d and 0"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + time.sleep(15) + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 11 end ...... ") + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + self.tmqCase10(cfgPath, buildPath) + self.tmqCase11(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From f78e329429ef2230aacef23acec954a2b1a8aea0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 17 Jun 2022 22:06:34 +0800 Subject: [PATCH 20/20] feat(taosAdapter): taosAdapter for 3.0 (#13951) Co-authored-by: t_max <1172915550@qq.com> --- tools/taosadapter | 1 + 1 file changed, 1 insertion(+) create mode 160000 tools/taosadapter diff --git a/tools/taosadapter b/tools/taosadapter new file mode 160000 index 0000000000..9ce3f5c98e --- /dev/null +++ b/tools/taosadapter @@ -0,0 +1 @@ +Subproject commit 9ce3f5c98ef95d9c7c596c4ed7302b0ed69a92b2