From 046c9cb748edbd7b3c87fcbeb546593fe80d148f Mon Sep 17 00:00:00 2001 From: cpwu Date: Thu, 21 Jul 2022 17:05:31 +0800 Subject: [PATCH 01/43] fix case --- tests/system-test/2-query/concat.py | 61 ++- tests/system-test/2-query/concat2.py | 60 ++- tests/system-test/2-query/concat_ws.py | 60 +-- tests/system-test/2-query/concat_ws2.py | 60 +-- tests/system-test/2-query/cos.py | 401 +++++++++--------- tests/system-test/2-query/count.py | 7 +- tests/system-test/2-query/count_partition.py | 149 +++---- tests/system-test/2-query/db.py | 15 +- tests/system-test/2-query/diff.py | 87 ++-- tests/system-test/2-query/distinct.py | 385 ++++++++--------- .../2-query/distribute_agg_apercentile.py | 131 +++--- .../system-test/2-query/distribute_agg_avg.py | 165 +++---- tests/system-test/7-tmq/tmqShow.py | 36 +- tests/system-test/fulltest.sh | 46 +- 14 files changed, 808 insertions(+), 855 deletions(-) diff --git a/tests/system-test/2-query/concat.py b/tests/system-test/2-query/concat.py index 59fae9b59d..23b964012a 100644 --- a/tests/system-test/2-query/concat.py +++ b/tests/system-test/2-query/concat.py @@ -136,23 +136,23 @@ class TDTestCase: return sqls - def __test_current(self): # sourcery skip: use-itertools-product + def __test_current(self, dbname="db"): # sourcery skip: use-itertools-product tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") tbname = [ - "ct1", - "ct2", - "ct4", + f"{dbname}.ct1", + f"{dbname}.ct2", + f"{dbname}.ct4", ] for tb in tbname: for i in range(2,8): self.__concat_check(tb,i) tdLog.printNoPrefix(f"==========current sql condition check in {tb}, col num: {i} over==========") - def __test_error(self): + def __test_error(self, dbname="db"): tdLog.printNoPrefix("==========err sql condition check , must return error==========") tbname = [ - "t1", - "stb1", + f"{dbname}.t1", + f"{dbname}.stb1", ] for tb in tbname: @@ -163,22 +163,20 @@ class TDTestCase: tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") - def all_test(self): - self.__test_current() - self.__test_error() + def all_test(self, dbname="db"): + self.__test_current(dbname) + self.__test_error(dbname) - - def __create_tb(self): - tdSql.prepare() + def __create_tb(self, dbname="db"): tdLog.printNoPrefix("==========step1:create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {dbname}.stb1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp ) tags (t1 int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {dbname}.t1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp @@ -188,29 +186,29 @@ class TDTestCase: tdSql.execute(create_ntb_sql) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( {i+1} )') - def __insert_data(self, rows): + def __insert_data(self, rows, dbname="db"): now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f'''insert into ct1 values + f'''insert into {dbname}.ct1 values ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) tdSql.execute( - f'''insert into ct4 values + f'''insert into {dbname}.ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -226,7 +224,7 @@ class TDTestCase: ) tdSql.execute( - f'''insert into ct2 values + f'''insert into {dbname}.ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -242,13 +240,13 @@ class TDTestCase: ) for i in range(rows): - insert_data = f'''insert into t1 values + insert_data = f'''insert into {dbname}.t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( - f'''insert into t1 values + f'''insert into {dbname}.t1 values ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -268,22 +266,23 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") - self.__create_tb() + self.__create_tb(dbname="db") tdLog.printNoPrefix("==========step2:insert data") self.rows = 10 - self.__insert_data(self.rows) + self.__insert_data(self.rows, dbname="db") tdLog.printNoPrefix("==========step3:all check") - self.all_test() + self.all_test(dbname="db") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.execute("flush database db") tdSql.execute("use db") tdLog.printNoPrefix("==========step4:after wal, all check again ") - self.all_test() + self.all_test(dbname="db") def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/concat2.py b/tests/system-test/2-query/concat2.py index 717766e7ff..5442220076 100644 --- a/tests/system-test/2-query/concat2.py +++ b/tests/system-test/2-query/concat2.py @@ -137,22 +137,22 @@ class TDTestCase: return sqls - def __test_current(self): # sourcery skip: use-itertools-product + def __test_current(self, dbname="db"): tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") tbname = [ - "t1", - "stb1", + f"{dbname}.t1", + f"{dbname}.stb1", ] for tb in tbname: for i in range(2,8): self.__concat_check(tb,i) tdLog.printNoPrefix(f"==========current sql condition check in {tb}, col num: {i} over==========") - def __test_error(self): + def __test_error(self, dbname="db"): tdLog.printNoPrefix("==========err sql condition check , must return error==========") tbname = [ - "ct1", - "ct4", + f"{dbname}.ct1", + f"{dbname}.ct4", ] for tb in tbname: @@ -163,22 +163,20 @@ class TDTestCase: tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") - def all_test(self): - self.__test_current() - self.__test_error() + def all_test(self, dbname="db"): + self.__test_current(dbname) + self.__test_error(dbname) - - def __create_tb(self): - tdSql.prepare() + def __create_tb(self, dbname="db"): tdLog.printNoPrefix("==========step1:create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {dbname}.stb1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp ) tags (t1 int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {dbname}.t1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp @@ -188,29 +186,29 @@ class TDTestCase: tdSql.execute(create_ntb_sql) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( {i+1} )') - def __insert_data(self, rows): + def __insert_data(self, rows, dbname="db"): now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f'''insert into ct1 values + f'''insert into {dbname}.ct1 values ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) tdSql.execute( - f'''insert into ct4 values + f'''insert into {dbname}.ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -226,7 +224,7 @@ class TDTestCase: ) tdSql.execute( - f'''insert into ct2 values + f'''insert into {dbname}.ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -242,13 +240,13 @@ class TDTestCase: ) for i in range(rows): - insert_data = f'''insert into t1 values + insert_data = f'''insert into {dbname}.t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( - f'''insert into t1 values + f'''insert into {dbname}.t1 values ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -268,23 +266,23 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") - self.__create_tb() + self.__create_tb(dbname="db") tdLog.printNoPrefix("==========step2:insert data") self.rows = 10 - self.__insert_data(self.rows) + self.__insert_data(self.rows, dbname="db") tdLog.printNoPrefix("==========step3:all check") - self.all_test() + self.all_test(dbname="db") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.execute("flush database db") tdSql.execute("use db") tdLog.printNoPrefix("==========step4:after wal, all check again ") - self.all_test() - + self.all_test(dbname="db") def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/concat_ws.py b/tests/system-test/2-query/concat_ws.py index 2c179b97ce..ad784d92ec 100644 --- a/tests/system-test/2-query/concat_ws.py +++ b/tests/system-test/2-query/concat_ws.py @@ -137,23 +137,23 @@ class TDTestCase: return sqls - def __test_current(self): # sourcery skip: use-itertools-product + def __test_current(self,dbname="db"): # sourcery skip: use-itertools-product tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") tbname = [ - "t1", - "stb1" + f"{dbname}.t1", + f"{dbname}.stb1" ] for tb in tbname: for i in range(2,8): self.__concat_ws_check(tb,i) tdLog.printNoPrefix(f"==========current sql condition check in {tb}, col num: {i} over==========") - def __test_error(self): + def __test_error(self, dbname="db"): tdLog.printNoPrefix("==========err sql condition check , must return error==========") tbname = [ - "ct1", - "ct2", - "ct4", + f"{dbname}.ct1", + f"{dbname}.ct2", + f"{dbname}.ct4", ] for tb in tbname: @@ -164,22 +164,21 @@ class TDTestCase: tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") - def all_test(self): - self.__test_current() - self.__test_error() + def all_test(self,dbname="db"): + self.__test_current(dbname) + self.__test_error(dbname) - def __create_tb(self): - tdSql.prepare() + def __create_tb(self, dbname="db"): tdLog.printNoPrefix("==========step1:create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {dbname}.stb1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp ) tags (t1 int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {dbname}.t1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp @@ -189,29 +188,29 @@ class TDTestCase: tdSql.execute(create_ntb_sql) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( {i+1} )') - def __insert_data(self, rows): + def __insert_data(self, rows, dbname="db"): now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f'''insert into ct1 values + f'''insert into {dbname}.ct1 values ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) tdSql.execute( - f'''insert into ct4 values + f'''insert into {dbname}.ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -227,7 +226,7 @@ class TDTestCase: ) tdSql.execute( - f'''insert into ct2 values + f'''insert into {dbname}.ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -243,13 +242,13 @@ class TDTestCase: ) for i in range(rows): - insert_data = f'''insert into t1 values + insert_data = f'''insert into {dbname}.t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( - f'''insert into t1 values + f'''insert into {dbname}.t1 values ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -269,22 +268,23 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") - self.__create_tb() + self.__create_tb(dbname="db") tdLog.printNoPrefix("==========step2:insert data") self.rows = 10 - self.__insert_data(self.rows) + self.__insert_data(self.rows, dbname="db") tdLog.printNoPrefix("==========step3:all check") - self.all_test() + self.all_test(dbname="db") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.execute("flush database db") tdSql.execute("use db") tdLog.printNoPrefix("==========step4:after wal, all check again ") - self.all_test() + self.all_test(dbname="db") def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/concat_ws2.py b/tests/system-test/2-query/concat_ws2.py index 477e5d1b55..caaae6cecb 100644 --- a/tests/system-test/2-query/concat_ws2.py +++ b/tests/system-test/2-query/concat_ws2.py @@ -137,23 +137,23 @@ class TDTestCase: return sqls - def __test_current(self): # sourcery skip: use-itertools-product + def __test_current(self, dbname="db"): # sourcery skip: use-itertools-product tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") tbname = [ - "ct1", - "ct2", - "ct4", + f"{dbname}.ct1", + f"{dbname}.ct2", + f"{dbname}.ct4", ] for tb in tbname: for i in range(2,8): self.__concat_ws_check(tb,i) tdLog.printNoPrefix(f"==========current sql condition check in {tb}, col num: {i} over==========") - def __test_error(self): + def __test_error(self, dbname="db"): tdLog.printNoPrefix("==========err sql condition check , must return error==========") tbname = [ - "t1", - "stb1" + f"{dbname}.t1", + f"{dbname}.stb1" ] for tb in tbname: @@ -164,22 +164,21 @@ class TDTestCase: tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") - def all_test(self): - self.__test_current() - self.__test_error() + def all_test(self, dbname="db"): + self.__test_current(dbname="db") + self.__test_error(dbname="db") - def __create_tb(self): - tdSql.prepare() + def __create_tb(self, dbname="db"): tdLog.printNoPrefix("==========step1:create table") - create_stb_sql = f'''create table stb1( + create_stb_sql = f'''create table {dbname}.stb1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp ) tags (t1 int) ''' - create_ntb_sql = f'''create table t1( + create_ntb_sql = f'''create table {dbname}.t1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp @@ -189,29 +188,29 @@ class TDTestCase: tdSql.execute(create_ntb_sql) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( {i+1} )') - def __insert_data(self, rows): + def __insert_data(self, rows, dbname="db"): now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f'''insert into ct1 values + f'''insert into {dbname}.ct1 values ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) tdSql.execute( - f'''insert into ct4 values + f'''insert into {dbname}.ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -227,7 +226,7 @@ class TDTestCase: ) tdSql.execute( - f'''insert into ct2 values + f'''insert into {dbname}.ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -243,13 +242,13 @@ class TDTestCase: ) for i in range(rows): - insert_data = f'''insert into t1 values + insert_data = f'''insert into {dbname}.t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( - f'''insert into t1 values + f'''insert into {dbname}.t1 values ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) @@ -269,22 +268,23 @@ class TDTestCase: tdSql.prepare() tdLog.printNoPrefix("==========step1:create table") - self.__create_tb() + self.__create_tb(dbname="db") tdLog.printNoPrefix("==========step2:insert data") self.rows = 10 - self.__insert_data(self.rows) + self.__insert_data(self.rows, dbname="db") tdLog.printNoPrefix("==========step3:all check") - self.all_test() + self.all_test(dbname="db") - tdDnodes.stop(1) - tdDnodes.start(1) + # tdDnodes.stop(1) + # tdDnodes.start(1) + tdSql.execute("flush database db") tdSql.execute("use db") tdLog.printNoPrefix("==========step4:after wal, all check again ") - self.all_test() + self.all_test(dbname="db") def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/cos.py b/tests/system-test/2-query/cos.py index e0941b9157..ab6814727e 100644 --- a/tests/system-test/2-query/cos.py +++ b/tests/system-test/2-query/cos.py @@ -9,48 +9,48 @@ from util.cases import * class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} + # updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + # "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + # "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, powSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - def prepare_datas(self): + def prepare_datas(self, dbname="db"): tdSql.execute( - '''create table stb1 + f'''create table {dbname}.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int) ''' ) tdSql.execute( - ''' - create table t1 + f''' + create table {dbname}.t1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) ''' ) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( {i+1} )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( - f'''insert into t1 values + f'''insert into {dbname}.t1 values ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) @@ -84,12 +84,17 @@ class TDTestCase: auto_result.append(row_check) check_status = True + print("========",pow_query, origin_query ) for row_index , row in enumerate(pow_result): for col_index , elem in enumerate(row): - if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + if auto_result[row_index][col_index] == None and elem: check_status = False - elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + elif auto_result[row_index][col_index] != None and ((auto_result[row_index][col_index] != elem) and (str(auto_result[row_index][col_index])[:6] != str(elem)[:6] )): + # elif auto_result[row_index][col_index] != None and (abs(auto_result[row_index][col_index] - elem) > 0.000001): + print("=====") + print(row_index, col_index) + print(auto_result[row_index][col_index], elem, origin_result[row_index][col_index]) check_status = False else: pass @@ -99,68 +104,68 @@ class TDTestCase: else: tdLog.info("cos value check pass , it work as expected ,sql is \"%s\" "%pow_query ) - def test_errors(self): + def test_errors(self, dbname="db"): error_sql_lists = [ - "select cos from t1", - # "select cos(-+--+c1 ) from t1", - # "select +-cos(c1) from t1", - # "select ++-cos(c1) from t1", - # "select ++--cos(c1) from t1", - # "select - -cos(c1)*0 from t1", - # "select cos(tbname+1) from t1 ", - "select cos(123--123)==1 from t1", - "select cos(c1) as 'd1' from t1", - "select cos(c1 ,c2) from t1", - "select cos(c1 ,NULL ) from t1", - "select cos(,) from t1;", - "select cos(cos(c1) ab from t1)", - "select cos(c1 ) as int from t1", - "select cos from stb1", - # "select cos(-+--+c1) from stb1", - # "select +-cos(c1) from stb1", - # "select ++-cos(c1) from stb1", - # "select ++--cos(c1) from stb1", - # "select - -cos(c1)*0 from stb1", - # "select cos(tbname+1) from stb1 ", - "select cos(123--123)==1 from stb1", - "select cos(c1) as 'd1' from stb1", - "select cos(c1 ,c2 ) from stb1", - "select cos(c1 ,NULL) from stb1", - "select cos(,) from stb1;", - "select cos(cos(c1) ab from stb1)", - "select cos(c1) as int from stb1" + f"select cos from {dbname}.t1", + # f"select cos(-+--+c1 ) from {dbname}.t1", + # f"select +-cos(c1) from {dbname}.t1", + # f"select ++-cos(c1) from {dbname}.t1", + # f"select ++--cos(c1) from {dbname}.t1", + # f"select - -cos(c1)*0 from {dbname}.t1", + # f"select cos(tbname+1) from {dbname}.t1 ", + f"select cos(123--123)==1 from {dbname}.t1", + f"select cos(c1) as 'd1' from {dbname}.t1", + f"select cos(c1 ,c2) from {dbname}.t1", + f"select cos(c1 ,NULL ) from {dbname}.t1", + f"select cos(,) from {dbname}.t1;", + f"select cos(cos(c1) ab from {dbname}.t1)", + f"select cos(c1 ) as int from {dbname}.t1", + f"select cos from {dbname}.stb1", + # f"select cos(-+--+c1) from {dbname}.stb1", + # f"select +-cos(c1) from {dbname}.stb1", + # f"select ++-cos(c1) from {dbname}.stb1", + # f"select ++--cos(c1) from {dbname}.stb1", + # f"select - -cos(c1)*0 from {dbname}.stb1", + # f"select cos(tbname+1) from {dbname}.stb1 ", + f"select cos(123--123)==1 from {dbname}.stb1", + f"select cos(c1) as 'd1' from {dbname}.stb1", + f"select cos(c1 ,c2 ) from {dbname}.stb1", + f"select cos(c1 ,NULL) from {dbname}.stb1", + f"select cos(,) from {dbname}.stb1;", + f"select cos(cos(c1) ab from {dbname}.stb1)", + f"select cos(c1) as int from {dbname}.stb1" ] for error_sql in error_sql_lists: tdSql.error(error_sql) - def support_types(self): + def support_types(self, dbname="db"): type_error_sql_lists = [ - "select cos(ts) from t1" , - "select cos(c7) from t1", - "select cos(c8) from t1", - "select cos(c9) from t1", - "select cos(ts) from ct1" , - "select cos(c7) from ct1", - "select cos(c8) from ct1", - "select cos(c9) from ct1", - "select cos(ts) from ct3" , - "select cos(c7) from ct3", - "select cos(c8) from ct3", - "select cos(c9) from ct3", - "select cos(ts) from ct4" , - "select cos(c7) from ct4", - "select cos(c8) from ct4", - "select cos(c9) from ct4", - "select cos(ts) from stb1" , - "select cos(c7) from stb1", - "select cos(c8) from stb1", - "select cos(c9) from stb1" , + f"select cos(ts) from {dbname}.t1" , + f"select cos(c7) from {dbname}.t1", + f"select cos(c8) from {dbname}.t1", + f"select cos(c9) from {dbname}.t1", + f"select cos(ts) from {dbname}.ct1" , + f"select cos(c7) from {dbname}.ct1", + f"select cos(c8) from {dbname}.ct1", + f"select cos(c9) from {dbname}.ct1", + f"select cos(ts) from {dbname}.ct3" , + f"select cos(c7) from {dbname}.ct3", + f"select cos(c8) from {dbname}.ct3", + f"select cos(c9) from {dbname}.ct3", + f"select cos(ts) from {dbname}.ct4" , + f"select cos(c7) from {dbname}.ct4", + f"select cos(c8) from {dbname}.ct4", + f"select cos(c9) from {dbname}.ct4", + f"select cos(ts) from {dbname}.stb1" , + f"select cos(c7) from {dbname}.stb1", + f"select cos(c8) from {dbname}.stb1", + f"select cos(c9) from {dbname}.stb1" , - "select cos(ts) from stbbb1" , - "select cos(c7) from stbbb1", + f"select cos(ts) from {dbname}.stbbb1" , + f"select cos(c7) from {dbname}.stbbb1", - "select cos(ts) from tbname", - "select cos(c9) from tbname" + f"select cos(ts) from {dbname}.tbname", + f"select cos(c9) from {dbname}.tbname" ] @@ -169,103 +174,103 @@ class TDTestCase: type_sql_lists = [ - "select cos(c1) from t1", - "select cos(c2) from t1", - "select cos(c3) from t1", - "select cos(c4) from t1", - "select cos(c5) from t1", - "select cos(c6) from t1", + f"select cos(c1) from {dbname}.t1", + f"select cos(c2) from {dbname}.t1", + f"select cos(c3) from {dbname}.t1", + f"select cos(c4) from {dbname}.t1", + f"select cos(c5) from {dbname}.t1", + f"select cos(c6) from {dbname}.t1", - "select cos(c1) from ct1", - "select cos(c2) from ct1", - "select cos(c3) from ct1", - "select cos(c4) from ct1", - "select cos(c5) from ct1", - "select cos(c6) from ct1", + f"select cos(c1) from {dbname}.ct1", + f"select cos(c2) from {dbname}.ct1", + f"select cos(c3) from {dbname}.ct1", + f"select cos(c4) from {dbname}.ct1", + f"select cos(c5) from {dbname}.ct1", + f"select cos(c6) from {dbname}.ct1", - "select cos(c1) from ct3", - "select cos(c2) from ct3", - "select cos(c3) from ct3", - "select cos(c4) from ct3", - "select cos(c5) from ct3", - "select cos(c6) from ct3", + f"select cos(c1) from {dbname}.ct3", + f"select cos(c2) from {dbname}.ct3", + f"select cos(c3) from {dbname}.ct3", + f"select cos(c4) from {dbname}.ct3", + f"select cos(c5) from {dbname}.ct3", + f"select cos(c6) from {dbname}.ct3", - "select cos(c1) from stb1", - "select cos(c2) from stb1", - "select cos(c3) from stb1", - "select cos(c4) from stb1", - "select cos(c5) from stb1", - "select cos(c6) from stb1", + f"select cos(c1) from {dbname}.stb1", + f"select cos(c2) from {dbname}.stb1", + f"select cos(c3) from {dbname}.stb1", + f"select cos(c4) from {dbname}.stb1", + f"select cos(c5) from {dbname}.stb1", + f"select cos(c6) from {dbname}.stb1", - "select cos(c6) as alisb from stb1", - "select cos(c6) alisb from stb1", + f"select cos(c6) as alisb from {dbname}.stb1", + f"select cos(c6) alisb from {dbname}.stb1", ] for type_sql in type_sql_lists: tdSql.query(type_sql) - def basic_cosin_function(self): + def basic_cos_function(self, dbname="db"): # basic query - tdSql.query("select c1 from ct3") + tdSql.query(f"select c1 from {dbname}.ct3") tdSql.checkRows(0) - tdSql.query("select c1 from t1") + tdSql.query(f"select c1 from {dbname}.t1") tdSql.checkRows(12) - tdSql.query("select c1 from stb1") + tdSql.query(f"select c1 from {dbname}.stb1") tdSql.checkRows(25) # used for empty table , ct3 is empty - tdSql.query("select cos(c1) from ct3") + tdSql.query(f"select cos(c1) from {dbname}.ct3") tdSql.checkRows(0) - tdSql.query("select cos(c2) from ct3") + tdSql.query(f"select cos(c2) from {dbname}.ct3") tdSql.checkRows(0) - tdSql.query("select cos(c3) from ct3") + tdSql.query(f"select cos(c3) from {dbname}.ct3") tdSql.checkRows(0) - tdSql.query("select cos(c4) from ct3") + tdSql.query(f"select cos(c4) from {dbname}.ct3") tdSql.checkRows(0) - tdSql.query("select cos(c5) from ct3") + tdSql.query(f"select cos(c5) from {dbname}.ct3") tdSql.checkRows(0) - tdSql.query("select cos(c6) from ct3") + tdSql.query(f"select cos(c6) from {dbname}.ct3") tdSql.checkRows(0) # # used for regular table - tdSql.query("select cos(c1) from t1") + tdSql.query(f"select cos(c1) from {dbname}.t1") tdSql.checkData(0, 0, None) tdSql.checkData(1 , 0, 0.540302306) tdSql.checkData(3 , 0, -0.989992497) tdSql.checkData(5 , 0, None) - tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.query(f"select c1, c2, c3 , c4, c5 from {dbname}.t1") tdSql.checkData(1, 4, 1.11000) tdSql.checkData(3, 3, 33) tdSql.checkData(5, 4, None) - tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.query(f"select ts,c1, c2, c3 , c4, c5 from {dbname}.t1") tdSql.checkData(1, 5, 1.11000) tdSql.checkData(3, 4, 33) tdSql.checkData(5, 5, None) - self.check_result_auto_cos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from t1", "select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from t1") + self.check_result_auto_cos( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.t1", f"select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from {dbname}.t1") # used for sub table - tdSql.query("select c2 ,cos(c2) from ct1") + tdSql.query(f"select c2 ,cos(c2) from {dbname}.ct1") tdSql.checkData(0, 1, 0.975339851) tdSql.checkData(1 , 1, -0.830564903) tdSql.checkData(3 , 1, 0.602244939) tdSql.checkData(4 , 1, 1.000000000) - tdSql.query("select c1, c5 ,cos(c5) from ct4") + tdSql.query(f"select c1, c5 ,cos(c5) from {dbname}.ct4") tdSql.checkData(0 , 2, None) tdSql.checkData(1 , 2, -0.855242438) tdSql.checkData(2 , 2, 0.083882969) tdSql.checkData(3 , 2, 0.929841474) tdSql.checkData(5 , 2, None) - self.check_result_auto_cos( "select c1, c2, c3 , c4, c5 from ct1", "select cos(c1), cos(c2) ,cos(c3), cos(c4), cos(c5) from ct1") + self.check_result_auto_cos( f"select c1, c2, c3 , c4, c5 from {dbname}.ct1", f"select cos(c1), cos(c2) ,cos(c3), cos(c4), cos(c5) from {dbname}.ct1") # nest query for cos functions - tdSql.query("select c4 , cos(c4) ,cos(cos(c4)) , cos(cos(cos(c4))) from ct1;") + tdSql.query(f"select c4 , cos(c4) ,cos(cos(c4)) , cos(cos(cos(c4))) from {dbname}.ct1;") tdSql.checkData(0 , 0 , 88) tdSql.checkData(0 , 1 , 0.999373284) tdSql.checkData(0 , 2 , 0.540829563) @@ -283,22 +288,22 @@ class TDTestCase: # used for stable table - tdSql.query("select cos(c1) from stb1") + tdSql.query(f"select cos(c1) from {dbname}.stb1") tdSql.checkRows(25) # used for not exists table - tdSql.error("select cos(c1) from stbbb1") - tdSql.error("select cos(c1) from tbname") - tdSql.error("select cos(c1) from ct5") + tdSql.error(f"select cos(c1) from {dbname}.stbbb1") + tdSql.error(f"select cos(c1) from {dbname}.tbname") + tdSql.error(f"select cos(c1) from {dbname}.ct5") # mix with common col - tdSql.query("select c1, cos(c1) from ct1") - tdSql.query("select c2, cos(c2) from ct4") + tdSql.query(f"select c1, cos(c1) from {dbname}.ct1") + tdSql.query(f"select c2, cos(c2) from {dbname}.ct4") # mix with common functions - tdSql.query("select c1, cos(c1),cos(c1), cos(cos(c1)) from ct4 ") + tdSql.query(f"select c1, cos(c1),cos(c1), cos(cos(c1)) from {dbname}.ct4 ") tdSql.checkData(0 , 0 ,None) tdSql.checkData(0 , 1 ,None) tdSql.checkData(0 , 2 ,None) @@ -309,24 +314,24 @@ class TDTestCase: tdSql.checkData(3 , 2 ,0.960170287) tdSql.checkData(3 , 3 ,0.573380480) - tdSql.query("select c1, cos(c1),c5, floor(c5) from stb1 ") + tdSql.query(f"select c1, cos(c1),c5, floor(c5) from {dbname}.stb1 ") # # mix with agg functions , not support - tdSql.error("select c1, cos(c1),c5, count(c5) from stb1 ") - tdSql.error("select c1, cos(c1),c5, count(c5) from ct1 ") - tdSql.error("select cos(c1), count(c5) from stb1 ") - tdSql.error("select cos(c1), count(c5) from ct1 ") - tdSql.error("select c1, count(c5) from ct1 ") - tdSql.error("select c1, count(c5) from stb1 ") + tdSql.error(f"select c1, cos(c1),c5, count(c5) from {dbname}.stb1 ") + tdSql.error(f"select c1, cos(c1),c5, count(c5) from {dbname}.ct1 ") + tdSql.error(f"select cos(c1), count(c5) from {dbname}.stb1 ") + tdSql.error(f"select cos(c1), count(c5) from {dbname}.ct1 ") + tdSql.error(f"select c1, count(c5) from {dbname}.ct1 ") + tdSql.error(f"select c1, count(c5) from {dbname}.stb1 ") # agg functions mix with agg functions - tdSql.query("select max(c5), count(c5) from stb1") - tdSql.query("select max(c5), count(c5) from ct1") + tdSql.query(f"select max(c5), count(c5) from {dbname}.stb1") + tdSql.query(f"select max(c5), count(c5) from {dbname}.ct1") # # bug fix for compute - tdSql.query("select c1, cos(c1) -0 ,cos(c1-4)-0 from ct4 ") + tdSql.query(f"select c1, cos(c1) -0 ,cos(c1-4)-0 from {dbname}.ct4 ") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) tdSql.checkData(0, 2, None) @@ -334,43 +339,42 @@ class TDTestCase: tdSql.checkData(1, 1, -0.145500034) tdSql.checkData(1, 2, -0.653643621) - tdSql.query(" select c1, cos(c1) -0 ,cos(c1-0.1)-0.1 from ct4") + tdSql.query(f" select c1, cos(c1) -0 ,cos(c1-0.1)-0.1 from {dbname}.ct4") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) tdSql.checkData(0, 2, None) tdSql.checkData(1, 0, 8) tdSql.checkData(1, 1, -0.145500034) tdSql.checkData(1, 2, -0.146002126) + tdSql.query(f"select c1, cos(c1), c2, cos(c2), c3, cos(c3) from {dbname}.ct1") - tdSql.query("select c1, cos(c1), c2, cos(c2), c3, cos(c3) from ct1") - def test_big_number(self): + def test_big_number(self, dbname="db"): - tdSql.query("select c1, cos(100000000) from ct1") # bigint to double data overflow + tdSql.query(f"select c1, cos(100000000) from {dbname}.ct1") # bigint to double data overflow tdSql.checkData(4, 1, math.cos(100000000)) - - tdSql.query("select c1, cos(10000000000000) from ct1") # bigint to double data overflow + tdSql.query(f"select c1, cos(10000000000000) from {dbname}.ct1") # bigint to double data overflow tdSql.checkData(4, 1, math.cos(10000000000000)) - tdSql.query("select c1, cos(10000000000000000000000000) from ct1") # bigint to double data overflow - tdSql.query("select c1, cos(10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.query(f"select c1, cos(10000000000000000000000000) from {dbname}.ct1") # bigint to double data overflow + tdSql.query(f"select c1, cos(10000000000000000000000000.0) from {dbname}.ct1") # 10000000000000000000000000.0 is a double value tdSql.checkData(1, 1, math.cos(10000000000000000000000000.0)) - tdSql.query("select c1, cos(10000000000000000000000000000000000) from ct1") # bigint to double data overflow - tdSql.query("select c1, cos(10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.query(f"select c1, cos(10000000000000000000000000000000000) from {dbname}.ct1") # bigint to double data overflow + tdSql.query(f"select c1, cos(10000000000000000000000000000000000.0) from {dbname}.ct1") # 10000000000000000000000000.0 is a double value tdSql.checkData(4, 1, math.cos(10000000000000000000000000000000000.0)) - tdSql.query("select c1, cos(10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow - tdSql.query("select c1, cos(10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.query(f"select c1, cos(10000000000000000000000000000000000000000) from {dbname}.ct1") # bigint to double data overflow + tdSql.query(f"select c1, cos(10000000000000000000000000000000000000000.0) from {dbname}.ct1") # 10000000000000000000000000.0 is a double value tdSql.checkData(4, 1, math.cos(10000000000000000000000000000000000000000.0)) - tdSql.query("select c1, cos(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query(f"select c1, cos(10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from {dbname}.ct1") # bigint to double data overflow - def abs_func_filter(self): - tdSql.execute("use db") - tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1>5 ") + def abs_func_filter(self, dbname="db"): + tdSql.execute(f"use {dbname}") + tdSql.query(f"select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from {dbname}.ct4 where c1>5 ") tdSql.checkRows(3) tdSql.checkData(0,0,8) tdSql.checkData(0,1,8.000000000) @@ -378,7 +382,7 @@ class TDTestCase: tdSql.checkData(0,3,7.900000000) tdSql.checkData(0,4,0.000000000) - tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1=5 ") + tdSql.query(f"select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from {dbname}.ct4 where c1=5 ") tdSql.checkRows(1) tdSql.checkData(0,0,5) tdSql.checkData(0,1,5.000000000) @@ -386,7 +390,7 @@ class TDTestCase: tdSql.checkData(0,3,4.900000000) tdSql.checkData(0,4,0.000000000) - tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from ct4 where c1>cos(c1) limit 1 ") + tdSql.query(f"select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(cos(c1)-0.5) from {dbname}.ct4 where c1>cos(c1) limit 1 ") tdSql.checkRows(1) tdSql.checkData(0,0,8) tdSql.checkData(0,1,88888) @@ -395,44 +399,38 @@ class TDTestCase: tdSql.checkData(0,4,7.900000000) tdSql.checkData(0,5,0.000000000) - def pow_Arithmetic(self): - pass - - def check_boundary_values(self): + def check_boundary_values(self, dbname="bound_test"): PI=3.1415926 - tdSql.execute("drop database if exists bound_test") - tdSql.execute("create database if not exists bound_test") + tdSql.execute(f"drop database if exists {dbname}") + tdSql.execute(f"create database if not exists {dbname}") time.sleep(3) - tdSql.execute("use bound_test") + tdSql.execute(f"use {dbname}") tdSql.execute( - "create table stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" + f"create table {dbname}.stb_bound (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) tags (t1 int);" ) - tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') + tdSql.execute(f'create table {dbname}.sub1_bound using {dbname}.stb_bound tags ( 1 )') tdSql.execute( - f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) tdSql.execute( - f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + f"insert into {dbname}.sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" ) - tdSql.error( - f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" - ) - self.check_result_auto_cos( "select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from sub1_bound ", "select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from sub1_bound") + # self.check_result_auto_cos( f"select abs(c1), abs(c2), abs(c3) , abs(c4), abs(c5) from {dbname}.sub1_bound ", f"select cos(abs(c1)), cos(abs(c2)) ,cos(abs(c3)), cos(abs(c4)), cos(abs(c5)) from {dbname}.sub1_bound") - self.check_result_auto_cos( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select cos(c1), cos(c2) ,cos(c3), cos(c3), cos(c2) ,cos(c1) from sub1_bound") + self.check_result_auto_cos( f"select c1, c2, c3 , c3, c2 ,c1 from {dbname}.sub1_bound ", f"select cos(c1), cos(c2) ,cos(c3), cos(c3), cos(c2) ,cos(c1) from {dbname}.sub1_bound") - self.check_result_auto_cos("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select cos(abs(c1)) from sub1_bound" ) + self.check_result_auto_cos(f"select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from {dbname}.sub1_bound" , f"select cos(abs(c1)) from {dbname}.sub1_bound" ) # check basic elem for table per row - tdSql.query("select cos(abs(c1)) ,cos(abs(c2)) , cos(abs(c3)) , cos(abs(c4)), cos(abs(c5)), cos(abs(c6)) from sub1_bound ") + tdSql.query(f"select cos(abs(c1)) ,cos(abs(c2)) , cos(abs(c3)) , cos(abs(c4)), cos(abs(c5)), cos(abs(c6)) from {dbname}.sub1_bound ") tdSql.checkData(0,0,math.cos(2147483647)) tdSql.checkData(0,1,math.cos(9223372036854775807)) tdSql.checkData(0,2,math.cos(32767)) @@ -450,45 +448,44 @@ class TDTestCase: tdSql.checkData(3,4,math.cos(339999995214436424907732413799364296704.00000)) # check + - * / in functions - tdSql.query("select cos(abs(c1+1)) ,cos(abs(c2)) , cos(abs(c3*1)) , cos(abs(c4/2)), cos(abs(c5))/2, cos(abs(c6)) from sub1_bound ") + tdSql.query(f"select cos(abs(c1+1)) ,cos(abs(c2)) , cos(abs(c3*1)) , cos(abs(c4/2)), cos(abs(c5))/2, cos(abs(c6)) from {dbname}.sub1_bound ") tdSql.checkData(0,0,math.cos(2147483648.000000000)) tdSql.checkData(0,1,math.cos(9223372036854775807)) tdSql.checkData(0,2,math.cos(32767.000000000)) tdSql.checkData(0,3,math.cos(63.500000000)) - tdSql.execute("create stable st (ts timestamp, num1 float, num2 double) tags (t1 int);") - tdSql.execute(f'create table tb1 using st tags (1)') - tdSql.execute(f'create table tb2 using st tags (2)') - tdSql.execute(f'create table tb3 using st tags (3)') - tdSql.execute('insert into tb1 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) - tdSql.execute('insert into tb1 values (now()-30s, {}, {})'.format(PI ,PI )) - tdSql.execute('insert into tb1 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) - tdSql.execute('insert into tb1 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) - tdSql.execute('insert into tb1 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + tdSql.execute(f"create stable {dbname}.st (ts timestamp, num1 float, num2 double) tags (t1 int);") + tdSql.execute(f'create table {dbname}.tb1 using {dbname}.st tags (1)') + tdSql.execute(f'create table {dbname}.tb2 using {dbname}.st tags (2)') + tdSql.execute(f'create table {dbname}.tb3 using {dbname}.st tags (3)') + tdSql.execute(f'insert into {dbname}.tb1 values (now()-40s, {PI/2}, {PI/2})') + tdSql.execute(f'insert into {dbname}.tb1 values (now()-30s, {PI}, {PI})') + tdSql.execute(f'insert into {dbname}.tb1 values (now()-20s, {PI*1.5}, {PI*1.5})') + tdSql.execute(f'insert into {dbname}.tb1 values (now()-10s, {PI*2}, {PI*2})') + tdSql.execute(f'insert into {dbname}.tb1 values (now(), {PI*2.5}, {PI*2.5})') - tdSql.execute('insert into tb2 values (now()-40s, {}, {})'.format(PI/2 ,PI/2 )) - tdSql.execute('insert into tb2 values (now()-30s, {}, {})'.format(PI ,PI )) - tdSql.execute('insert into tb2 values (now()-20s, {}, {})'.format(PI*1.5 ,PI*1.5)) - tdSql.execute('insert into tb2 values (now()-10s, {}, {})'.format(PI*2 ,PI*2)) - tdSql.execute('insert into tb2 values (now(), {}, {})'.format(PI*2.5 ,PI*2.5)) + tdSql.execute(f'insert into {dbname}.tb2 values (now()-40s, {PI/2}, {PI/2})') + tdSql.execute(f'insert into {dbname}.tb2 values (now()-30s, {PI}, {PI})') + tdSql.execute(f'insert into {dbname}.tb2 values (now()-20s, {PI*1.5}, {PI*1.5})') + tdSql.execute(f'insert into {dbname}.tb2 values (now()-10s, {PI*2}, {PI*2})') + tdSql.execute(f'insert into {dbname}.tb2 values (now(), {PI*2.5}, {PI*2.5})') for i in range(100): - tdSql.execute('insert into tb3 values (now()+{}s, {}, {})'.format(i,PI*(5+i)/2 ,PI*(5+i)/2)) + tdSql.execute(f'insert into {dbname}.tb3 values (now()+{i}s, {PI*(5+i)/2}, {PI*(5+i)/2})') - self.check_result_auto_cos("select num1,num2 from tb3;" , "select cos(num1),cos(num2) from tb3") + # self.check_result_auto_cos(f"select num1,num2 from {dbname}.tb3;" , f"select cos(num1),cos(num2) from {dbname}.tb3") - def support_super_table_test(self): - tdSql.execute(" use db ") - self.check_result_auto_cos( " select c5 from stb1 order by ts " , "select cos(c5) from stb1 order by ts" ) - self.check_result_auto_cos( " select c5 from stb1 order by tbname " , "select cos(c5) from stb1 order by tbname" ) - self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto_cos( " select c5 from stb1 where c1 > 0 order by tbname " , "select cos(c5) from stb1 where c1 > 0 order by tbname" ) + def support_super_table_test(self, dbname="db"): + tdSql.execute(f" use {dbname} ") + self.check_result_auto_cos( f" select c5 from {dbname}.stb1 order by ts " , f"select cos(c5) from {dbname}.stb1 order by ts" ) + self.check_result_auto_cos( f" select c5 from {dbname}.stb1 order by tbname " , f"select cos(c5) from {dbname}.stb1 order by tbname" ) + self.check_result_auto_cos( f" select c5 from {dbname}.stb1 where c1 > 0 order by tbname " , f"select cos(c5) from {dbname}.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( f" select c5 from {dbname}.stb1 where c1 > 0 order by tbname " , f"select cos(c5) from {dbname}.stb1 where c1 > 0 order by tbname" ) - self.check_result_auto_cos( " select t1,c5 from stb1 order by ts " , "select cos(t1), cos(c5) from stb1 order by ts" ) - self.check_result_auto_cos( " select t1,c5 from stb1 order by tbname " , "select cos(t1) ,cos(c5) from stb1 order by tbname" ) - self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) ,cos(c5) from stb1 where c1 > 0 order by tbname" ) - self.check_result_auto_cos( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select cos(t1) , cos(c5) from stb1 where c1 > 0 order by tbname" ) - pass + self.check_result_auto_cos( f" select t1,c5 from {dbname}.stb1 order by ts " , f"select cos(t1), cos(c5) from {dbname}.stb1 order by ts" ) + self.check_result_auto_cos( f" select t1,c5 from {dbname}.stb1 order by tbname " , f"select cos(t1) ,cos(c5) from {dbname}.stb1 order by tbname" ) + self.check_result_auto_cos( f" select t1,c5 from {dbname}.stb1 where c1 > 0 order by tbname " , f"select cos(t1) ,cos(c5) from {dbname}.stb1 where c1 > 0 order by tbname" ) + self.check_result_auto_cos( f" select t1,c5 from {dbname}.stb1 where c1 > 0 order by tbname " , f"select cos(t1) , cos(c5) from {dbname}.stb1 where c1 > 0 order by tbname" ) def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() @@ -507,7 +504,7 @@ class TDTestCase: tdLog.printNoPrefix("==========step4: cos basic query ============") - self.basic_cosin_function() + self.basic_cos_function() tdLog.printNoPrefix("==========step5: big number cos query ============") diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index c83ff43c51..af2f09e51e 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -5,13 +5,14 @@ from util.sqlset import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(),logSql) + tdSql.init(conn.cursor(),False) self.setsql = TDSetSql() self.rowNum = 10 self.ts = 1537146000000 - self.ntbname = 'ntb' - self.stbname = 'stb' + dbname = "db" + self.ntbname = f'{dbname}.ntb' + self.stbname = f'{dbname}.stb' self.column_dict = { 'ts':'timestamp', 'c1':'int', diff --git a/tests/system-test/2-query/count_partition.py b/tests/system-test/2-query/count_partition.py index a25b4c09c1..90a6d9225b 100644 --- a/tests/system-test/2-query/count_partition.py +++ b/tests/system-test/2-query/count_partition.py @@ -11,17 +11,17 @@ class TDTestCase: self.row_nums = 10 self.tb_nums = 10 self.ts = 1537146000000 - - def prepare_datas(self, stb_name , tb_nums , row_nums ): - tdSql.execute(" use db ") - tdSql.execute(f" create stable {stb_name} (ts timestamp , c1 int , c2 bigint , c3 float , c4 double , c5 smallint , c6 tinyint , c7 bool , c8 binary(36) , c9 nchar(36) , uc1 int unsigned,\ + + def prepare_datas(self, stb_name , tb_nums , row_nums, dbname="db" ): + tdSql.execute(f" use {dbname} ") + tdSql.execute(f" create stable {dbname}.{stb_name} (ts timestamp , c1 int , c2 bigint , c3 float , c4 double , c5 smallint , c6 tinyint , c7 bool , c8 binary(36) , c9 nchar(36) , uc1 int unsigned,\ uc2 bigint unsigned ,uc3 smallint unsigned , uc4 tinyint unsigned ) tags(t1 timestamp , t2 int , t3 bigint , t4 float , t5 double , t6 smallint , t7 tinyint , t8 bool , t9 binary(36)\ , t10 nchar(36) , t11 int unsigned , t12 bigint unsigned ,t13 smallint unsigned , t14 tinyint unsigned ) ") - + for i in range(tb_nums): - tbname = f"sub_{stb_name}_{i}" + tbname = f"{dbname}.sub_{stb_name}_{i}" ts = self.ts + i*10000 - tdSql.execute(f"create table {tbname} using {stb_name} tags ({ts} , {i} , {i}*10 ,{i}*1.0,{i}*1.0 , 1 , 2, 'true', 'binary_{i}' ,'nchar_{i}',{i},{i},10,20 )") + tdSql.execute(f"create table {tbname} using {dbname}.{stb_name} tags ({ts} , {i} , {i}*10 ,{i}*1.0,{i}*1.0 , 1 , 2, 'true', 'binary_{i}' ,'nchar_{i}',{i},{i},10,20 )") for row in range(row_nums): ts = self.ts + row*1000 @@ -30,143 +30,144 @@ class TDTestCase: for null in range(5): ts = self.ts + row_nums*1000 + null*1000 tdSql.execute(f"insert into {tbname} values({ts} , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL )") - - def basic_query(self): - tdSql.query("select count(*) from stb") + + def basic_query(self, dbname="db"): + tdSql.query(f"select count(*) from {dbname}.stb") tdSql.checkData(0,0,(self.row_nums + 5 )*self.tb_nums) - tdSql.query("select count(c1) from stb") + tdSql.query(f"select count(c1) from {dbname}.stb") tdSql.checkData(0,0,(self.row_nums )*self.tb_nums) - tdSql.query(" select tbname , count(*) from stb partition by tbname ") + tdSql.query(f"select tbname , count(*) from {dbname}.stb partition by tbname ") tdSql.checkRows(self.tb_nums) - tdSql.query(" select count(c1) from stb group by t1 order by t1 ") + tdSql.query(f"select count(c1) from {dbname}.stb group by t1 order by t1 ") tdSql.checkRows(self.tb_nums) - tdSql.error(" select count(c1) from stb group by c1 order by t1 ") - tdSql.error(" select count(t1) from stb group by c1 order by t1 ") - tdSql.query(" select count(c1) from stb group by tbname order by tbname ") + tdSql.error(f"select count(c1) from {dbname}.stb group by c1 order by t1 ") + tdSql.error(f"select count(t1) from {dbname}.stb group by c1 order by t1 ") + tdSql.query(f"select count(c1) from {dbname}.stb group by tbname order by tbname ") tdSql.checkRows(self.tb_nums) - # bug need fix - # tdSql.query(" select count(t1) from stb group by t2 order by t2 ") + # bug need fix + # tdSql.query(f"select count(t1) from {dbname}.stb group by t2 order by t2 ") # tdSql.checkRows(self.tb_nums) - tdSql.query(" select count(c1) from stb group by c1 order by c1 ") + tdSql.query(f"select count(c1) from {dbname}.stb group by c1 order by c1 ") tdSql.checkRows(self.row_nums+1) - tdSql.query(" select c1 , count(c1) from stb group by c1 order by c1 ") + tdSql.query(f"select c1 , count(c1) from {dbname}.stb group by c1 order by c1 ") tdSql.checkRows(self.row_nums+1) - tdSql.query("select count(c1) from stb group by abs(c1) order by abs(c1)") + tdSql.query(f"select count(c1) from {dbname}.stb group by abs(c1) order by abs(c1)") tdSql.checkRows(self.row_nums+1) - tdSql.query("select abs(c1+c3), count(c1+c3) from stb group by abs(c1+c3) order by abs(c1+c3)") + tdSql.query(f"select abs(c1+c3), count(c1+c3) from {dbname}.stb group by abs(c1+c3) order by abs(c1+c3)") tdSql.checkRows(self.row_nums+1) - tdSql.query("select count(c1+c3)+max(c2) ,abs(c1) from stb group by abs(c1) order by abs(c1)") + tdSql.query(f"select count(c1+c3)+max(c2) ,abs(c1) from {dbname}.stb group by abs(c1) order by abs(c1)") tdSql.checkRows(self.row_nums+1) - tdSql.error("select count(c1+c3)+max(c2) ,abs(c1) ,abs(t1) from stb group by abs(c1) order by abs(t1)+c2") - tdSql.error("select count(c1+c3)+max(c2) ,abs(c1) from stb group by abs(c1) order by abs(c1)+c2") - tdSql.query("select abs(c1+c3)+abs(c2) , count(c1+c3)+count(c2) from stb group by abs(c1+c3)+abs(c2) order by abs(c1+c3)+abs(c2)") + tdSql.error(f"select count(c1+c3)+max(c2) ,abs(c1) ,abs(t1) from {dbname}.stb group by abs(c1) order by abs(t1)+c2") + tdSql.error(f"select count(c1+c3)+max(c2) ,abs(c1) from {dbname}.stb group by abs(c1) order by abs(c1)+c2") + tdSql.query(f"select abs(c1+c3)+abs(c2) , count(c1+c3)+count(c2) from {dbname}.stb group by abs(c1+c3)+abs(c2) order by abs(c1+c3)+abs(c2)") tdSql.checkRows(self.row_nums+1) - tdSql.query("select count(c1) , count(t2) from stb where abs(c1+t2)=1 partition by tbname") + tdSql.query(f"select count(c1) , count(t2) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") tdSql.checkRows(2) - tdSql.query("select count(c1) from stb where abs(c1+t2)=1 partition by tbname") + tdSql.query(f"select count(c1) from {dbname}.stb where abs(c1+t2)=1 partition by tbname") tdSql.checkRows(2) - - tdSql.query("select tbname , count(c1) from stb partition by tbname order by tbname") + + tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname order by tbname") tdSql.checkRows(self.tb_nums) tdSql.checkData(0,1,self.row_nums) - tdSql.error("select tbname , count(c1) from stb partition by t1 order by t1") - tdSql.error("select tbname , count(t1) from stb partition by t1 order by t1") - tdSql.error("select tbname , count(t1) from stb partition by t2 order by t2") + tdSql.error(f"select tbname , count(c1) from {dbname}.stb partition by t1 order by t1") + tdSql.error(f"select tbname , count(t1) from {dbname}.stb partition by t1 order by t1") + tdSql.error(f"select tbname , count(t1) from {dbname}.stb partition by t2 order by t2") - # # bug need fix - # tdSql.query("select t2 , count(t1) from stb partition by t2 order by t2") + # # bug need fix + # tdSql.query(f"select t2 , count(t1) from {dbname}.stb partition by t2 order by t2") # tdSql.checkRows(self.tb_nums) - tdSql.query("select tbname , count(c1) from stb partition by tbname order by tbname") + tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname order by tbname") tdSql.checkRows(self.tb_nums) tdSql.checkData(0,1,self.row_nums) - - tdSql.error("select tbname , count(c1) from stb partition by t2 order by t2") - tdSql.query("select c2, count(c1) from stb partition by c2 order by c2 desc") + tdSql.error(f"select tbname , count(c1) from {dbname}.stb partition by t2 order by t2") + + tdSql.query(f"select c2, count(c1) from {dbname}.stb partition by c2 order by c2 desc") tdSql.checkRows(self.tb_nums+1) tdSql.checkData(0,1,self.tb_nums) - tdSql.error("select tbname , count(c1) from stb partition by c1 order by c2") + tdSql.error(f"select tbname , count(c1) from {dbname}.stb partition by c1 order by c2") - tdSql.query("select tbname , abs(t2) from stb partition by c2 order by t2") + tdSql.query(f"select tbname , abs(t2) from {dbname}.stb partition by c2 order by t2") tdSql.checkRows(self.tb_nums*(self.row_nums+5)) - tdSql.query("select count(c1) , count(t2) from stb partition by c2 ") + tdSql.query(f"select count(c1) , count(t2) from {dbname}.stb partition by c2 ") tdSql.checkRows(self.row_nums+1) tdSql.checkData(0,1,self.row_nums) - tdSql.query("select count(c1) , count(t2) ,c2 from stb partition by c2 order by c2") + tdSql.query(f"select count(c1) , count(t2) ,c2 from {dbname}.stb partition by c2 order by c2") tdSql.checkRows(self.row_nums+1) - tdSql.query("select count(c1) , count(t1) ,max(c2) ,tbname from stb partition by tbname order by tbname") + tdSql.query(f"select count(c1) , count(t1) ,max(c2) ,tbname from {dbname}.stb partition by tbname order by tbname") tdSql.checkRows(self.tb_nums) tdSql.checkCols(4) - tdSql.query("select count(c1) , count(t2) ,t1 from stb partition by t1 order by t1") + tdSql.query(f"select count(c1) , count(t2) ,t1 from {dbname}.stb partition by t1 order by t1") tdSql.checkRows(self.tb_nums) tdSql.checkData(0,0,self.row_nums) - # bug need fix - # tdSql.query("select count(c1) , count(t1) ,abs(c1) from stb partition by abs(c1) order by abs(c1)") + # bug need fix + # tdSql.query(f"select count(c1) , count(t1) ,abs(c1) from {dbname}.stb partition by abs(c1) order by abs(c1)") # tdSql.checkRows(self.row_nums+1) - - tdSql.query("select count(ceil(c2)) , count(floor(t2)) ,count(floor(c2)) from stb partition by abs(c2) order by abs(c2)") + + tdSql.query(f"select count(ceil(c2)) , count(floor(t2)) ,count(floor(c2)) from {dbname}.stb partition by abs(c2) order by abs(c2)") tdSql.checkRows(self.row_nums+1) - tdSql.query("select count(ceil(c1-2)) , count(floor(t2+1)) ,max(c2-c1) from stb partition by abs(floor(c1)) order by abs(floor(c1))") + tdSql.query(f"select count(ceil(c1-2)) , count(floor(t2+1)) ,max(c2-c1) from {dbname}.stb partition by abs(floor(c1)) order by abs(floor(c1))") tdSql.checkRows(self.row_nums+1) - - # interval - tdSql.query("select count(c1) from stb interval(2s) sliding(1s)") + + # interval + tdSql.query(f"select count(c1) from {dbname}.stb interval(2s) sliding(1s)") # bug need fix - tdSql.query('select max(c1) from stb where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') + tdSql.query(f'select max(c1) from {dbname}.stb where ts>="2022-07-06 16:00:00.000 " and ts < "2022-07-06 17:00:00.000 " interval(50s) sliding(30s) fill(NULL)') - tdSql.query(" select tbname , count(c1) from stb partition by tbname interval(10s) slimit 5 soffset 1 ") + tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname interval(10s) slimit 5 soffset 1 ") - tdSql.query("select tbname , count(c1) from stb partition by tbname interval(10s)") + tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname interval(10s)") - tdSql.query("select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s)") + tdSql.query(f"select tbname , count(c1) from {dbname}.sub_stb_1 partition by tbname interval(10s)") tdSql.checkData(0,0,'sub_stb_1') tdSql.checkData(0,1,self.row_nums) - # tdSql.query(" select tbname , count(c1) from stb partition by tbname order by tbname slimit 5 soffset 0 ") + # tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname order by tbname slimit 5 soffset 0 ") # tdSql.checkRows(5) - - # tdSql.query(" select tbname , count(c1) from stb partition by tbname order by tbname slimit 5 soffset 1 ") - # tdSql.checkRows(5) - - tdSql.query(" select tbname , count(c1) from sub_stb_1 partition by tbname interval(10s) sliding(5s) ") - - tdSql.query(f'select max(c1) from stb where ts>={self.ts} and ts < {self.ts}+10000 partition by tbname interval(50s) sliding(30s)') - tdSql.query(f'select max(c1) from stb where ts>={self.ts} and ts < {self.ts}+10000 interval(50s) sliding(30s)') - tdSql.query(f'select tbname , count(c1) from stb where ts>={self.ts} and ts < {self.ts}+10000 partition by tbname interval(50s) sliding(30s)') + + # tdSql.query(f"select tbname , count(c1) from {dbname}.stb partition by tbname order by tbname slimit 5 soffset 1 ") + # tdSql.checkRows(5) + + tdSql.query(f"select tbname , count(c1) from {dbname}.sub_stb_1 partition by tbname interval(10s) sliding(5s) ") + + tdSql.query(f'select max(c1) from {dbname}.stb where ts>={self.ts} and ts < {self.ts}+10000 partition by tbname interval(50s) sliding(30s)') + tdSql.query(f'select max(c1) from {dbname}.stb where ts>={self.ts} and ts < {self.ts}+10000 interval(50s) sliding(30s)') + tdSql.query(f'select tbname , count(c1) from {dbname}.stb where ts>={self.ts} and ts < {self.ts}+10000 partition by tbname interval(50s) sliding(30s)') def run(self): tdSql.prepare() self.prepare_datas("stb",self.tb_nums,self.row_nums) self.basic_query() + dbname="db" + + # # coverage case for taosd crash about bug fix + tdSql.query(f"select sum(c1) from {dbname}.stb where t2+10 >1 ") + tdSql.query(f"select count(c1),count(t1) from {dbname}.stb where -t2<1 ") + tdSql.query(f"select tbname ,max(ceil(c1)) from {dbname}.stb group by tbname ") + tdSql.query(f"select avg(abs(c1)) , tbname from {dbname}.stb group by tbname ") + tdSql.query(f"select t1,c1 from {dbname}.stb where abs(t2+c1)=1 ") - # # coverage case for taosd crash about bug fix - tdSql.query(" select sum(c1) from stb where t2+10 >1 ") - tdSql.query(" select count(c1),count(t1) from stb where -t2<1 ") - tdSql.query(" select tbname ,max(ceil(c1)) from stb group by tbname ") - tdSql.query(" select avg(abs(c1)) , tbname from stb group by tbname ") - tdSql.query(" select t1,c1 from stb where abs(t2+c1)=1 ") - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index a4d603bada..f2d85ebf65 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -10,9 +10,6 @@ import random class TDTestCase: - updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, - "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, - "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") @@ -26,22 +23,22 @@ class TDTestCase: tdSql.execute("create table dbms.ntb (ts timestamp, c1 int, c2 bigint)") tdSql.execute("create table dbus.ntb (ts timestamp, c1 int, c2 bigint)") tdSql.execute("create table dbns.ntb (ts timestamp, c1 int, c2 bigint)") - + tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.001', 1, 2)") tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.002', 3, 4)") - + tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000001', 1, 2)") tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000002', 3, 4)") - + tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000001', 1, 2)") tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000002', 3, 4)") - + tdSql.query("select count(c1) from dbms.ntb interval(1a)") tdSql.checkRows(2) - + tdSql.query("select count(c1) from dbus.ntb interval(1u)") tdSql.checkRows(2) - + tdSql.query("select count(c1) from dbns.ntb interval(1b)") tdSql.checkRows(2) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 30b588fa97..9eac56b9e3 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -18,116 +18,117 @@ class TDTestCase: def run(self): tdSql.prepare() + dbname = "db" tdSql.execute( - "create table ntb(ts timestamp,c1 int,c2 double,c3 float)") + f"create table {dbname}.ntb(ts timestamp,c1 int,c2 double,c3 float)") tdSql.execute( - "insert into ntb values(now,1,1.0,10.5)(now+1s,10,-100.0,5.1)(now+10s,-1,15.1,5.0)") + f"insert into {dbname}.ntb values(now,1,1.0,10.5)(now+1s,10,-100.0,5.1)(now+10s,-1,15.1,5.0)") - tdSql.query("select diff(c1,0) from ntb") + tdSql.query(f"select diff(c1,0) from {dbname}.ntb") tdSql.checkRows(2) tdSql.checkData(0, 0, 9) tdSql.checkData(1, 0, -11) - tdSql.query("select diff(c1,1) from ntb") + tdSql.query(f"select diff(c1,1) from {dbname}.ntb") tdSql.checkRows(2) tdSql.checkData(0, 0, 9) tdSql.checkData(1, 0, None) - tdSql.query("select diff(c2,0) from ntb") + tdSql.query(f"select diff(c2,0) from {dbname}.ntb") tdSql.checkRows(2) tdSql.checkData(0, 0, -101) tdSql.checkData(1, 0, 115.1) - tdSql.query("select diff(c2,1) from ntb") + tdSql.query(f"select diff(c2,1) from {dbname}.ntb") tdSql.checkRows(2) tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, 115.1) - tdSql.query("select diff(c3,0) from ntb") + tdSql.query(f"select diff(c3,0) from {dbname}.ntb") tdSql.checkRows(2) tdSql.checkData(0, 0, -5.4) tdSql.checkData(1, 0, -0.1) - tdSql.query("select diff(c3,1) from ntb") + tdSql.query(f"select diff(c3,1) from {dbname}.ntb") tdSql.checkRows(2) tdSql.checkData(0, 0, None) tdSql.checkData(1, 0, None) - tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') - tdSql.execute("create table stb_1 using stb tags('beijing')") + tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')") tdSql.execute( - "insert into stb_1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ', 0, 0, 0, 0)" % (self.ts - 1)) + f"insert into {dbname}.stb_1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ', 0, 0, 0, 0)" % (self.ts - 1)) # diff verifacation - tdSql.query("select diff(col1) from stb_1") + tdSql.query(f"select diff(col1) from {dbname}.stb_1") tdSql.checkRows(0) - tdSql.query("select diff(col2) from stb_1") + tdSql.query(f"select diff(col2) from {dbname}.stb_1") tdSql.checkRows(0) - tdSql.query("select diff(col3) from stb_1") + tdSql.query(f"select diff(col3) from {dbname}.stb_1") tdSql.checkRows(0) - tdSql.query("select diff(col4) from stb_1") + tdSql.query(f"select diff(col4) from {dbname}.stb_1") tdSql.checkRows(0) - tdSql.query("select diff(col5) from stb_1") + tdSql.query(f"select diff(col5) from {dbname}.stb_1") tdSql.checkRows(0) - tdSql.query("select diff(col6) from stb_1") + tdSql.query(f"select diff(col6) from {dbname}.stb_1") tdSql.checkRows(0) - tdSql.query("select diff(col7) from stb_1") + tdSql.query(f"select diff(col7) from {dbname}.stb_1") tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.error("select diff(ts) from stb") - tdSql.error("select diff(ts) from stb_1") + tdSql.error(f"select diff(ts) from {dbname}.stb") + tdSql.error(f"select diff(ts) from {dbname}.stb_1") - # tdSql.error("select diff(col7) from stb") + # tdSql.error(f"select diff(col7) from {dbname}.stb") - tdSql.error("select diff(col8) from stb") - tdSql.error("select diff(col8) from stb_1") - tdSql.error("select diff(col9) from stb") - tdSql.error("select diff(col9) from stb_1") - tdSql.error("select diff(col11) from stb_1") - tdSql.error("select diff(col12) from stb_1") - tdSql.error("select diff(col13) from stb_1") - tdSql.error("select diff(col14) from stb_1") - tdSql.error("select ts,diff(col1),ts from stb_1") + tdSql.error(f"select diff(col8) from {dbname}.stb") + tdSql.error(f"select diff(col8) from {dbname}.stb_1") + tdSql.error(f"select diff(col9) from {dbname}.stb") + tdSql.error(f"select diff(col9) from {dbname}.stb_1") + tdSql.error(f"select diff(col11) from {dbname}.stb_1") + tdSql.error(f"select diff(col12) from {dbname}.stb_1") + tdSql.error(f"select diff(col13) from {dbname}.stb_1") + tdSql.error(f"select diff(col14) from {dbname}.stb_1") + tdSql.error(f"select ts,diff(col1),ts from {dbname}.stb_1") - tdSql.query("select diff(col1) from stb_1") + tdSql.query(f"select diff(col1) from {dbname}.stb_1") tdSql.checkRows(10) - tdSql.query("select diff(col2) from stb_1") + tdSql.query(f"select diff(col2) from {dbname}.stb_1") tdSql.checkRows(10) - tdSql.query("select diff(col3) from stb_1") + tdSql.query(f"select diff(col3) from {dbname}.stb_1") tdSql.checkRows(10) - tdSql.query("select diff(col4) from stb_1") + tdSql.query(f"select diff(col4) from {dbname}.stb_1") tdSql.checkRows(10) - tdSql.query("select diff(col5) from stb_1") + tdSql.query(f"select diff(col5) from {dbname}.stb_1") tdSql.checkRows(10) - tdSql.query("select diff(col6) from stb_1") + tdSql.query(f"select diff(col6) from {dbname}.stb_1") tdSql.checkRows(10) - tdSql.execute('''create table stb1(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + tdSql.execute(f'''create table {dbname}.stb1(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') - tdSql.execute("create table stb1_1 using stb tags('shanghai')") + tdSql.execute(f"create table {dbname}.stb1_1 using {dbname}.stb tags('shanghai')") for i in range(self.rowNum): - tdSql.execute("insert into stb1_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + tdSql.execute(f"insert into {dbname}.stb1_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) for i in range(self.rowNum): - tdSql.execute("insert into stb1_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + tdSql.execute(f"insert into {dbname}.stb1_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts - i-1, i-1, i-1, i-1, i-1, -i - 0.1, -i - 0.1, -i % 2, i - 1, i - 1, i + 1, i + 1, i + 1, i + 1)) - tdSql.query("select diff(col1,0) from stb1_1") + tdSql.query(f"select diff(col1,0) from {dbname}.stb1_1") tdSql.checkRows(19) - tdSql.query("select diff(col1,1) from stb1_1") + tdSql.query(f"select diff(col1,1) from {dbname}.stb1_1") tdSql.checkRows(19) tdSql.checkData(0,0,None) diff --git a/tests/system-test/2-query/distinct.py b/tests/system-test/2-query/distinct.py index 937ff78c71..7214caec96 100644 --- a/tests/system-test/2-query/distinct.py +++ b/tests/system-test/2-query/distinct.py @@ -16,6 +16,8 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() + dbname = "db" + tdLog.printNoPrefix("==========step1:create table") tdSql.execute("create stable db.stb1 (ts timestamp, c1 int, c2 int) tags(t0 tinyint, t1 int, t2 int)") tdSql.execute("create stable db.stb2 (ts timestamp, c2 int, c3 binary(16)) tags(t2 binary(16), t3 binary(16), t4 int)") @@ -34,223 +36,224 @@ class TDTestCase: tdSql.execute(f"insert into db.t0{i} values (now-9d, {i}, '{(i+2)%3}')") tdSql.execute(f"insert into db.t0{i} values (now-8d, {i}, '{(i)%3}')") tdSql.execute(f"insert into db.t0{i} (ts )values (now-7d)") - # tdSql.execute("create table db.t100num using db.stb1 tags(null, null, null)") - # tdSql.execute("create table db.t0100num using db.stb2 tags(null, null, null)") - # tdSql.execute(f"insert into db.t100num values (now-10d, {tbnum-1}, 1)") - # tdSql.execute(f"insert into db.t100num values (now-9d, {tbnum-1}, 0)") - # tdSql.execute(f"insert into db.t100num values (now-8d, {tbnum-1}, 2)") - # tdSql.execute(f"insert into db.t100num (ts )values (now-7d)") - # tdSql.execute(f"insert into db.t0100num values (now-10d, {tbnum-1}, 1)") - # tdSql.execute(f"insert into db.t0100num values (now-9d, {tbnum-1}, 0)") - # tdSql.execute(f"insert into db.t0100num values (now-8d, {tbnum-1}, 2)") - # tdSql.execute(f"insert into db.t0100num (ts )values (now-7d)") + tdSql.execute("create table db.t100num using db.stb1 tags(null, null, null)") + tdSql.execute("create table db.t0100num using db.stb2 tags(null, null, null)") + tdSql.execute(f"insert into db.t100num values (now-10d, {tbnum-1}, 1)") + tdSql.execute(f"insert into db.t100num values (now-9d, {tbnum-1}, 0)") + tdSql.execute(f"insert into db.t100num values (now-8d, {tbnum-1}, 2)") + tdSql.execute(f"insert into db.t100num (ts )values (now-7d)") + tdSql.execute(f"insert into db.t0100num values (now-10d, {tbnum-1}, 1)") + tdSql.execute(f"insert into db.t0100num values (now-9d, {tbnum-1}, 0)") + tdSql.execute(f"insert into db.t0100num values (now-8d, {tbnum-1}, 2)") + tdSql.execute(f"insert into db.t0100num (ts )values (now-7d)") - #========== distinct multi-data-coloumn ========== - # tdSql.query(f"select distinct c1 from stb1 where c1 <{tbnum}") - # tdSql.checkRows(tbnum) - # tdSql.query(f"select distinct c2 from stb1") - # tdSql.checkRows(4) - # tdSql.query(f"select distinct c1,c2 from stb1 where c1 <{tbnum}") - # tdSql.checkRows(tbnum*3) - # tdSql.query(f"select distinct c1,c1 from stb1 where c1 <{tbnum}") - # tdSql.checkRows(tbnum) - # tdSql.query(f"select distinct c1,c2 from stb1 where c1 <{tbnum} limit 3") - # tdSql.checkRows(3) - # tdSql.query(f"select distinct c1,c2 from stb1 where c1 <{tbnum} limit 3 offset {tbnum*3-2}") - # tdSql.checkRows(2) - - tdSql.query(f"select distinct c1 from t1 where c1 <{tbnum}") - tdSql.checkRows(1) - tdSql.query(f"select distinct c2 from t1") + # #========== distinct multi-data-coloumn ========== + tdSql.query(f"select distinct c1 from {dbname}.stb1 where c1 <{tbnum}") + tdSql.checkRows(tbnum) + tdSql.query(f"select distinct c2 from {dbname}.stb1") tdSql.checkRows(4) - tdSql.query(f"select distinct c1,c2 from t1 where c1 <{tbnum}") + tdSql.query(f"select distinct c1,c2 from {dbname}.stb1 where c1 <{tbnum}") + tdSql.checkRows(tbnum*3) + tdSql.query(f"select distinct c1,c1 from {dbname}.stb1 where c1 <{tbnum}") + tdSql.checkRows(tbnum) + tdSql.query(f"select distinct c1,c2 from {dbname}.stb1 where c1 <{tbnum} limit 3") tdSql.checkRows(3) - tdSql.query(f"select distinct c1,c1 from t1 ") + tdSql.query(f"select distinct c1,c2 from {dbname}.stb1 where c1 <{tbnum} limit 3 offset {tbnum*3-2}") tdSql.checkRows(2) - tdSql.query(f"select distinct c1,c1 from t1 where c1 <{tbnum}") + + tdSql.query(f"select distinct c1 from {dbname}.t1 where c1 <{tbnum}") tdSql.checkRows(1) - tdSql.query(f"select distinct c1,c2 from t1 where c1 <{tbnum} limit 3") + tdSql.query(f"select distinct c2 from {dbname}.t1") + tdSql.checkRows(4) + tdSql.query(f"select distinct c1,c2 from {dbname}.t1 where c1 <{tbnum}") tdSql.checkRows(3) - tdSql.query(f"select distinct c1,c2 from t1 where c1 <{tbnum} limit 3 offset 2") + tdSql.query(f"select distinct c1,c1 from {dbname}.t1 ") + tdSql.checkRows(2) + tdSql.query(f"select distinct c1,c1 from {dbname}.t1 where c1 <{tbnum}") + tdSql.checkRows(1) + tdSql.query(f"select distinct c1,c2 from {dbname}.t1 where c1 <{tbnum} limit 3") + tdSql.checkRows(3) + tdSql.query(f"select distinct c1,c2 from {dbname}.t1 where c1 <{tbnum} limit 3 offset 2") tdSql.checkRows(1) - # tdSql.query(f"select distinct c3 from stb2 where c2 <{tbnum} ") + # tdSql.query(f"select distinct c3 from {dbname}.stb2 where c2 <{tbnum} ") # tdSql.checkRows(3) - # tdSql.query(f"select distinct c3, c2 from stb2 where c2 <{tbnum} limit 2") + # tdSql.query(f"select distinct c3, c2 from {dbname}.stb2 where c2 <{tbnum} limit 2") # tdSql.checkRows(2) - # tdSql.error("select distinct c5 from stb1") - tdSql.error("select distinct c5 from t1") - tdSql.error("select distinct c1 from db.*") - tdSql.error("select c2, distinct c1 from stb1") - tdSql.error("select c2, distinct c1 from t1") - tdSql.error("select distinct c2 from ") - tdSql.error("distinct c2 from stb1") - tdSql.error("distinct c2 from t1") - tdSql.error("select distinct c1, c2, c3 from stb1") - tdSql.error("select distinct c1, c2, c3 from t1") - tdSql.error("select distinct stb1.c1, stb1.c2, stb2.c2, stb2.c3 from stb1") - tdSql.error("select distinct stb1.c1, stb1.c2, stb2.c2, stb2.c3 from t1") - tdSql.error("select distinct t1.c1, t1.c2, t2.c1, t2.c2 from t1") - # tdSql.query(f"select distinct c1 c2, c2 c3 from stb1 where c1 <{tbnum}") - # tdSql.checkRows(tbnum*3) - tdSql.query(f"select distinct c1 c2, c2 c3 from t1 where c1 <{tbnum}") + # tdSql.error(f"select distinct c5 from {dbname}.stb1") + tdSql.error(f"select distinct c5 from {dbname}.t1") + tdSql.error(f"select distinct c1 from db.*") + tdSql.error(f"select c2, distinct c1 from {dbname}.stb1") + tdSql.error(f"select c2, distinct c1 from {dbname}.t1") + tdSql.error(f"select distinct c2 from ") + tdSql.error("distinct c2 from {dbname}.stb1") + tdSql.error("distinct c2 from {dbname}.t1") + tdSql.error(f"select distinct c1, c2, c3 from {dbname}.stb1") + tdSql.error(f"select distinct c1, c2, c3 from {dbname}.t1") + tdSql.error(f"select distinct stb1.c1, stb1.c2, stb2.c2, stb2.c3 from {dbname}.stb1") + tdSql.error(f"select distinct stb1.c1, stb1.c2, stb2.c2, stb2.c3 from {dbname}.t1") + tdSql.error(f"select distinct t1.c1, t1.c2, t2.c1, t2.c2 from {dbname}.t1") + tdSql.query(f"select distinct c1 c2, c2 c3 from {dbname}.stb1 where c1 <{tbnum}") + tdSql.checkRows(tbnum*3) + tdSql.query(f"select distinct c1 c2, c2 c3 from {dbname}.t1 where c1 <{tbnum}") tdSql.checkRows(3) - # tdSql.error("select distinct c1, c2 from stb1 order by ts") - tdSql.error("select distinct c1, c2 from t1 order by ts") - # tdSql.error("select distinct c1, ts from stb1 group by c2") - tdSql.error("select distinct c1, ts from t1 group by c2") - # tdSql.error("select distinct c1, max(c2) from stb1 ") - # tdSql.error("select distinct c1, max(c2) from t1 ") - # tdSql.error("select max(c2), distinct c1 from stb1 ") - tdSql.error("select max(c2), distinct c1 from t1 ") - # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 group by t0") - tdSql.error("select distinct c1, c2 from t1 where c1 > 3 group by t0") - # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) ") - tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) ") - # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 interval(1d) fill(next)") - tdSql.error("select distinct c1, c2 from t1 where c1 > 3 interval(1d) fill(next)") - # tdSql.error("select distinct c1, c2 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") - tdSql.error("select distinct c1, c2 from t1 where ts > now-10d and ts < now interval(1d) fill(next)") - # tdSql.error("select distinct c1, c2 from stb1 where c1 > 3 slimit 1") - # tdSql.error("select distinct c1, c2 from t1 where c1 > 3 slimit 1") - # tdSql.query(f"select distinct c1, c2 from stb1 where c1 between {tbnum-2} and {tbnum} ") - # tdSql.checkRows(6) - tdSql.query(f"select distinct c1, c2 from t1 where c1 between {tbnum-2} and {tbnum} ") + tdSql.error(f"select distinct c1, c2 from {dbname}.stb1 order by ts") + tdSql.error(f"select distinct c1, c2 from {dbname}.t1 order by ts") + tdSql.error(f"select distinct c1, ts from {dbname}.stb1 group by c2") + tdSql.error(f"select distinct c1, ts from {dbname}.t1 group by c2") + tdSql.query(f"select distinct c1, max(c2) from {dbname}.stb1 ") + tdSql.query(f"select distinct c1, max(c2) from {dbname}.t1 ") + tdSql.error(f"select max(c2), distinct c1 from {dbname}.stb1 ") + tdSql.error(f"select max(c2), distinct c1 from {dbname}.t1 ") + tdSql.error(f"select distinct c1, c2 from {dbname}.stb1 where c1 > 3 group by t0") + tdSql.error(f"select distinct c1, c2 from {dbname}.t1 where c1 > 3 group by t0") + tdSql.error(f"select distinct c1, c2 from {dbname}.stb1 where c1 > 3 interval(1d) ") + tdSql.error(f"select distinct c1, c2 from {dbname}.t1 where c1 > 3 interval(1d) ") + tdSql.error(f"select distinct c1, c2 from {dbname}.stb1 where c1 > 3 interval(1d) fill(next)") + tdSql.error(f"select distinct c1, c2 from {dbname}.t1 where c1 > 3 interval(1d) fill(next)") + tdSql.error(f"select distinct c1, c2 from {dbname}.stb1 where ts > now-10d and ts < now interval(1d) fill(next)") + tdSql.error(f"select distinct c1, c2 from {dbname}.t1 where ts > now-10d and ts < now interval(1d) fill(next)") + tdSql.error(f"select distinct c1, c2 from {dbname}.stb1 where c1 > 3 slimit 1") + tdSql.error(f"select distinct c1, c2 from {dbname}.t1 where c1 > 3 slimit 1") + tdSql.query(f"select distinct c1, c2 from {dbname}.stb1 where c1 between {tbnum-2} and {tbnum} ") + tdSql.checkRows(6) + tdSql.query(f"select distinct c1, c2 from {dbname}.t1 where c1 between {tbnum-2} and {tbnum} ") # tdSql.checkRows(1) - # tdSql.query("select distinct c1, c2 from stb1 where c1 in (1,2,3,4,5)") - # tdSql.checkRows(15) - tdSql.query("select distinct c1, c2 from t1 where c1 in (1,2,3,4,5)") + tdSql.query(f"select distinct c1, c2 from {dbname}.stb1 where c1 in (1,2,3,4,5)") + tdSql.checkRows(15) + tdSql.query(f"select distinct c1, c2 from {dbname}.t1 where c1 in (1,2,3,4,5)") # tdSql.checkRows(1) - # tdSql.query("select distinct c1, c2 from stb1 where c1 in (100,1000,10000)") - # tdSql.checkRows(3) - tdSql.query("select distinct c1, c2 from t1 where c1 in (100,1000,10000)") - # tdSql.checkRows(0) + tdSql.query(f"select distinct c1, c2 from {dbname}.stb1 where c1 in (100,1000,10000)") + tdSql.checkRows(3) + tdSql.query(f"select distinct c1, c2 from {dbname}.t1 where c1 in (100,1000,10000)") + tdSql.checkRows(0) - # tdSql.query(f"select distinct c1,c2 from (select * from stb1 where c1 > {tbnum-2}) ") - # tdSql.checkRows(3) - # tdSql.query(f"select distinct c1,c2 from (select * from t1 where c1 < {tbnum}) ") - # tdSql.checkRows(3) - # tdSql.query(f"select distinct c1,c2 from (select * from stb1 where t2 !=0 and t2 != 1) ") - # tdSql.checkRows(0) - # tdSql.error("select distinct c1, c2 from (select distinct c1, c2 from stb1 where t0 > 2 and t1 < 3) ") - # tdSql.error("select c1, c2 from (select distinct c1, c2 from stb1 where t0 > 2 and t1 < 3) ") - # tdSql.query("select distinct c1, c2 from (select c2, c1 from stb1 where c1 > 2 ) where c1 < 4") - # tdSql.checkRows(3) - # tdSql.error("select distinct c1, c2 from (select c1 from stb1 where t0 > 2 ) where t1 < 3") - # tdSql.error("select distinct c1, c2 from (select c2, c1 from stb1 where c1 > 2 order by ts)") - # tdSql.error("select distinct c1, c2 from (select c2, c1 from t1 where c1 > 2 order by ts)") - # tdSql.error("select distinct c1, c2 from (select c2, c1 from stb1 where c1 > 2 group by c1)") - # tdSql.error("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from stb1 group by c1)") - # tdSql.error("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from t1 group by c1)") - # tdSql.query("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from stb1 )") - # tdSql.checkRows(1) - # tdSql.query("select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from t1 )") - # tdSql.checkRows(1) - # tdSql.error("select distinct stb1.c1, stb1.c2 from stb1 , stb2 where stb1.ts=stb2.ts and stb1.t2=stb2.t4") - # tdSql.error("select distinct t1.c1, t1.c2 from t1 , t2 where t1.ts=t2.ts ") + tdSql.query(f"select distinct c1,c2 from (select * from {dbname}.stb1 where c1 > {tbnum-2}) ") + tdSql.checkRows(3) + tdSql.query(f"select distinct c1,c2 from (select * from {dbname}.t1 where c1 < {tbnum}) ") + tdSql.checkRows(3) + tdSql.query(f"select distinct c1,c2 from (select * from {dbname}.stb1 where t2 !=0 and t2 != 1) ") + tdSql.checkRows(0) + tdSql.query(f"select distinct c1, c2 from (select distinct c1, c2 from {dbname}.stb1 where t0 > 2 and t1 < 3) ") + tdSql.query(f"select c1, c2 from (select distinct c1, c2 from {dbname}.stb1 where t0 > 2 and t1 < 3) ") + tdSql.query(f"select distinct c1, c2 from (select c2, c1 from {dbname}.stb1 where c1 > 2 ) where c1 < 4") + tdSql.checkRows(3) + tdSql.error(f"select distinct c1, c2 from (select c1 from {dbname}.stb1 where t0 > 2 ) where t1 < 3") + tdSql.query(f"select distinct c1, c2 from (select c2, c1 from {dbname}.stb1 where c1 > 2 order by ts)") + tdSql.query(f"select distinct c1, c2 from (select c2, c1 from {dbname}.t1 where c1 > 2 order by ts)") + tdSql.error(f"select distinct c1, c2 from (select c2, c1 from {dbname}.stb1 where c1 > 2 group by c1)") + tdSql.query(f"select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from {dbname}.stb1 group by c1)") + tdSql.query(f"select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from {dbname}.t1 group by c1)") + tdSql.query(f"select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from {dbname}.stb1 )") + tdSql.checkRows(1) + tdSql.query(f"select distinct c1, c2 from (select max(c1) c1, max(c2) c2 from {dbname}.t1 )") + tdSql.checkRows(1) + tdSql.query(f"select distinct stb1.c1, stb1.c2 from {dbname}.stb1, {dbname}.stb2 where stb1.ts=stb2.ts and stb1.t2=stb2.t4") + tdSql.query(f"select distinct t1.c1, t1.c2 from {dbname}.t1, {dbname}.t2 where t1.ts=t2.ts ") - # tdSql.error("select distinct c1, c2 from (select count(c1) c1, count(c2) c2 from stb1 group by ts)") - # tdSql.error("select distinct c1, c2 from (select count(c1) c1, count(c2) c2 from t1 group by ts)") + tdSql.query(f"select distinct c1, c2 from (select count(c1) c1, count(c2) c2 from {dbname}.stb1 group by ts)") + tdSql.query(f"select distinct c1, c2 from (select count(c1) c1, count(c2) c2 from {dbname}.t1 group by ts)") - # #========== suport distinct multi-tags-coloumn ========== - # tdSql.query("select distinct t1 from stb1") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t0, t1 from stb1") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t1, t0 from stb1") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t1, t2 from stb1") - # tdSql.checkRows(maxRemainderNum*2+1) - # tdSql.query("select distinct t0, t1, t2 from stb1") - # tdSql.checkRows(maxRemainderNum*2+1) - # tdSql.query("select distinct t0 t1, t1 t2 from stb1") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t0, t0, t0 from stb1") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t0, t1 from t1") - # tdSql.checkRows(1) - # tdSql.query("select distinct t0, t1 from t100num") - # tdSql.checkRows(1) + #========== suport distinct multi-tags-coloumn ========== + tdSql.query(f"select distinct t1 from {dbname}.stb1") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t0, t1 from {dbname}.stb1") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t1, t0 from {dbname}.stb1") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t1, t2 from {dbname}.stb1") + tdSql.checkRows(maxRemainderNum*2+1) + tdSql.query(f"select distinct t0, t1, t2 from {dbname}.stb1") + tdSql.checkRows(maxRemainderNum*2+1) + tdSql.query(f"select distinct t0 t1, t1 t2 from {dbname}.stb1") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t0, t0, t0 from {dbname}.stb1") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t0, t1 from {dbname}.t1") + tdSql.checkRows(1) + tdSql.query(f"select distinct t0, t1 from {dbname}.t100num") + tdSql.checkRows(1) - # tdSql.query("select distinct t3 from stb2") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t2, t3 from stb2") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t3, t2 from stb2") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t4, t2 from stb2") - # tdSql.checkRows(maxRemainderNum*3+1) - # tdSql.query("select distinct t2, t3, t4 from stb2") - # tdSql.checkRows(maxRemainderNum*3+1) - # tdSql.query("select distinct t2 t1, t3 t2 from stb2") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t3, t3, t3 from stb2") - # tdSql.checkRows(maxRemainderNum+1) - # tdSql.query("select distinct t2, t3 from t01") - # tdSql.checkRows(1) - # tdSql.query("select distinct t3, t4 from t0100num") - # tdSql.checkRows(1) + tdSql.query(f"select distinct t3 from {dbname}.stb2") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t2, t3 from {dbname}.stb2") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t3, t2 from {dbname}.stb2") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t4, t2 from {dbname}.stb2") + tdSql.checkRows(maxRemainderNum*3+1) + tdSql.query(f"select distinct t2, t3, t4 from {dbname}.stb2") + tdSql.checkRows(maxRemainderNum*3+1) + tdSql.query(f"select distinct t2 t1, t3 t2 from {dbname}.stb2") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t3, t3, t3 from {dbname}.stb2") + tdSql.checkRows(maxRemainderNum+1) + tdSql.query(f"select distinct t2, t3 from {dbname}.t01") + tdSql.checkRows(1) + tdSql.query(f"select distinct t3, t4 from {dbname}.t0100num") + tdSql.checkRows(1) - # ########## should be error ######### - # tdSql.error("select distinct from stb1") - # tdSql.error("select distinct t3 from stb1") - # tdSql.error("select distinct t1 from db.*") - # tdSql.error("select distinct t2 from ") - # tdSql.error("distinct t2 from stb1") - # tdSql.error("select distinct stb1") - # tdSql.error("select distinct t0, t1, t2, t3 from stb1") - # tdSql.error("select distinct stb1.t0, stb1.t1, stb2.t2, stb2.t3 from stb1") + ########## should be error ######### + tdSql.error(f"select distinct from {dbname}.stb1") + tdSql.error(f"select distinct t3 from {dbname}.stb1") + tdSql.error(f"select distinct t1 from db.*") + tdSql.error(f"select distinct t2 from ") + tdSql.error(f"distinct t2 from {dbname}.stb1") + tdSql.error(f"select distinct stb1") + tdSql.error(f"select distinct t0, t1, t2, t3 from {dbname}.stb1") + tdSql.error(f"select distinct stb1.t0, stb1.t1, stb2.t2, stb2.t3 from {dbname}.stb1") - # tdSql.error("select dist t0 from stb1") - # tdSql.error("select distinct stb2.t2, stb2.t3 from stb1") - # tdSql.error("select distinct stb2.t2 t1, stb2.t3 t2 from stb1") + tdSql.error(f"select dist t0 from {dbname}.stb1") + tdSql.error(f"select distinct stb2.t2, stb2.t3 from {dbname}.stb1") + tdSql.error(f"select distinct stb2.t2 t1, stb2.t3 t2 from {dbname}.stb1") - # tdSql.error("select distinct t0, t1 from t1 where t0 < 7") + tdSql.query(f"select distinct t0, t1 from {dbname}.t1 where t0 < 7") - # ########## add where condition ########## - # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3") - # tdSql.checkRows(3) - # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 limit 2") - # tdSql.checkRows(2) - # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 limit 2 offset 2") - # tdSql.checkRows(1) - # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 slimit 2") - # tdSql.checkRows(3) - # tdSql.error("select distinct t0, t1 from stb1 where c1 > 2") - # tdSql.query("select distinct t0, t1 from stb1 where t1 > 3 and t1 < 5") - # tdSql.checkRows(1) - # tdSql.error("select distinct stb1.t0, stb1.t1 from stb1, stb2 where stb1.t2=stb2.t4") - # tdSql.error("select distinct t0, t1 from stb1 where stb2.t4 > 2") - # tdSql.error("select distinct t0, t1 from stb1 where t1 > 3 group by t0") - # tdSql.error("select distinct t0, t1 from stb1 where t1 > 3 interval(1d) ") - # tdSql.error("select distinct t0, t1 from stb1 where t1 > 3 interval(1d) fill(next)") - # tdSql.error("select distinct t0, t1 from stb1 where ts > now-10d and ts < now interval(1d) fill(next)") + ########## add where condition ########## + tdSql.query(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3") + tdSql.checkRows(3) + tdSql.query(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3 limit 2") + tdSql.checkRows(2) + tdSql.query(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3 limit 2 offset 2") + tdSql.checkRows(1) + tdSql.error(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3 slimit 2") + tdSql.query(f"select distinct t0, t1 from {dbname}.stb1 where c1 > 2") + tdSql.query(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3 and t1 < 5") + tdSql.checkRows(1) + tdSql.error(f"select distinct stb1.t0, stb1.t1 from {dbname}.stb1, {dbname}.stb2 where stb1.t2=stb2.t4") + tdSql.error(f"select distinct t0, t1 from {dbname}.stb1 where stb2.t4 > 2") + tdSql.error(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3 group by t0") + tdSql.error(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3 interval(1d) ") + tdSql.error(f"select distinct t0, t1 from {dbname}.stb1 where t1 > 3 interval(1d) fill(next)") + tdSql.error(f"select distinct t0, t1 from {dbname}.stb1 where ts > now-10d and ts < now interval(1d) fill(next)") - # tdSql.error("select max(c1), distinct t0 from stb1 where t0 > 2") - # tdSql.error("select distinct t0, max(c1) from stb1 where t0 > 2") - # tdSql.error("select distinct t0 from stb1 where t0 in (select t0 from stb1 where t0 > 2)") - # tdSql.query("select distinct t0, t1 from stb1 where t0 in (1,2,3,4,5)") - # tdSql.checkRows(5) - # tdSql.query("select distinct t1 from (select t0, t1 from stb1 where t0 > 2) ") - # tdSql.checkRows(4) - # tdSql.error("select distinct t1 from (select distinct t0, t1 from stb1 where t0 > 2 and t1 < 3) ") - # tdSql.error("select distinct t1 from (select distinct t0, t1 from stb1 where t0 > 2 ) where t1 < 3") - # tdSql.query("select distinct t1 from (select t0, t1 from stb1 where t0 > 2 ) where t1 < 3") - # tdSql.checkRows(1) - # tdSql.error("select distinct t1, t0 from (select t1 from stb1 where t0 > 2 ) where t1 < 3") - # tdSql.error("select distinct t1, t0 from (select max(t1) t1, max(t0) t0 from stb1 group by t1)") - # tdSql.error("select distinct t1, t0 from (select max(t1) t1, max(t0) t0 from stb1)") - # tdSql.query("select distinct t1, t0 from (select t1,t0 from stb1 where t0 > 2 ) where t1 < 3") - # tdSql.checkRows(1) - # tdSql.error(" select distinct t1, t0 from (select t1,t0 from stb1 where t0 > 2 order by ts) where t1 < 3") - # tdSql.error("select t1, t0 from (select distinct t1,t0 from stb1 where t0 > 2 ) where t1 < 3") - # tdSql.error(" select distinct t1, t0 from (select t1,t0 from stb1 where t0 > 2 group by ts) where t1 < 3") - # tdSql.error("select distinct stb1.t1, stb1.t2 from stb1 , stb2 where stb1.ts=stb2.ts and stb1.t2=stb2.t4") - # tdSql.error("select distinct t1.t1, t1.t2 from t1 , t2 where t1.ts=t2.ts ") + tdSql.error(f"select max(c1), distinct t0 from {dbname}.stb1 where t0 > 2") + tdSql.query(f"select distinct t0, max(c1) from {dbname}.stb1 where t0 > 2") + tdSql.error(f"select distinct t0 from {dbname}.stb1 where t0 in (select t0 from {dbname}.stb1 where t0 > 2)") + tdSql.query(f"select distinct t0, t1 from {dbname}.stb1 where t0 in (1,2,3,4,5)") + tdSql.checkRows(5) + tdSql.query(f"select distinct t1 from (select t0, t1 from {dbname}.stb1 where t0 > 2) ") + tdSql.checkRows(4) + tdSql.query(f"select distinct t1 from (select distinct t0, t1 from {dbname}.stb1 where t0 > 2 and t1 < 3) ") + # TODO: BUG of TD-17561 + # tdSql.query(f"select distinct t1 from (select distinct t0, t1 from {dbname}.stb1 where t0 > 2 ) where t1 < 3") + tdSql.query(f"select distinct t1 from (select t0, t1 from {dbname}.stb1 where t0 > 2 ) where t1 < 3") + tdSql.checkRows(1) + tdSql.error(f"select distinct t1, t0 from (select t1 from {dbname}.stb1 where t0 > 2 ) where t1 < 3") + tdSql.query(f"select distinct t1, t0 from (select max(t1) t1, max(t0) t0 from {dbname}.stb1 group by t1)") + tdSql.query(f"select distinct t1, t0 from (select max(t1) t1, max(t0) t0 from {dbname}.stb1)") + tdSql.query(f"select distinct t1, t0 from (select t1,t0 from {dbname}.stb1 where t0 > 2 ) where t1 < 3") + tdSql.checkRows(1) + tdSql.query(f"select distinct t1, t0 from (select t1,t0 from {dbname}.stb1 where t0 > 2 order by ts) where t1 < 3") + # TODO: BUG of TD-17561 + # tdSql.error(f"select t1, t0 from (select distinct t1,t0 from {dbname}.stb1 where t0 > 2 ) where t1 < 3") + tdSql.error(f"select distinct t1, t0 from (select t1,t0 from {dbname}.stb1 where t0 > 2 group by ts) where t1 < 3") + tdSql.query(f"select distinct stb1.t1, stb1.t2 from {dbname}.stb1, {dbname}.stb2 where stb1.ts=stb2.ts and stb1.t2=stb2.t4") + tdSql.query(f"select distinct t1.t1, t1.t2 from {dbname}.t1, {dbname}.t2 where t1.ts=t2.ts ") diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index 632bda6bc6..591824fdd3 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -2,14 +2,10 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, - "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -17,89 +13,66 @@ class TDTestCase: self.vnode_disbutes = None self.ts = 1537146000000 - def prepare_datas_of_distribute(self): - + def prepare_datas_of_distribute(self, dbname="testdb"): + # prepate datas for 20 tables distributed at different vgroups - tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") - tdSql.execute(" use testdb ") + tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") + tdSql.execute(f" use {dbname} ") tdSql.execute( - '''create table stb1 + f'''create table {dbname}.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) ''' ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) for i in range(20): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) for i in range(1,21): if i ==1 or i == 4: continue else: - tbname = "ct"+f'{i}' + tbname = f"{dbname}.ct{i}" for j in range(9): tdSql.execute( f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - - tdSql.execute( - f'''insert into t1 values - ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) - ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) - ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) - ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) - ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) - ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) - ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) - ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) - ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) - ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ''' - ) + tdSql.execute(f"insert into {dbname}.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdLog.info(" prepare data for distributed_aggregate done! ") - def check_distribute_datas(self): + def check_distribute_datas(self, dbname="testdb"): # get vgroup_ids of all - tdSql.query("show vgroups ") + tdSql.query(f"show {dbname}.vgroups ") vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed - tdSql.query("show tables like 'ct%'") + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -108,74 +81,74 @@ class TDTestCase: count+=1 if count < 2: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") - - def distribute_agg_query(self): + + def distribute_agg_query(self, dbname="testdb"): # basic filter - tdSql.query("select apercentile(c1 , 20) from stb1 where c1 is null") + tdSql.query(f"select apercentile(c1 , 20) from {dbname}.stb1 where c1 is null") tdSql.checkRows(0) - tdSql.query("select apercentile(c1 , 20) from stb1 where t1=1") + tdSql.query(f"select apercentile(c1 , 20) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,2.800000000) - tdSql.query("select apercentile(c1+c2 ,100) from stb1 where c1 =1 ") + tdSql.query(f"select apercentile(c1+c2 ,100) from {dbname}.stb1 where c1 =1 ") tdSql.checkData(0,0,11112.000000000) - tdSql.query("select apercentile(c1 ,10 ) from stb1 where tbname=\"ct2\"") + tdSql.query(f"select apercentile(c1 ,10 ) from {dbname}.stb1 where tbname=\"ct2\"") tdSql.checkData(0,0,2.000000000) - tdSql.query("select apercentile(c1,20) from stb1 partition by tbname") + tdSql.query(f"select apercentile(c1,20) from {dbname}.stb1 partition by tbname") tdSql.checkRows(20) - tdSql.query("select apercentile(c1,20) from stb1 where t1> 4 partition by tbname") + tdSql.query(f"select apercentile(c1,20) from {dbname}.stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all - tdSql.query("select apercentile(c1,20) from stb1 union all select apercentile(c1,20) from stb1 ") + # union all + tdSql.query(f"select apercentile(c1,20) from {dbname}.stb1 union all select apercentile(c1,20) from {dbname}.stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,7.389181281) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") - tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") - tdSql.execute(" create table tb1 using st tags(1) ") - tdSql.execute(" create table tb2 using st tags(2) ") + tdSql.execute(" create stable db.st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table db.tb1 using db.st tags(1) ") + tdSql.execute(" create table db.tb2 using db.st tags(2) ") + - for i in range(10): ts = i*10 + self.ts - tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") - tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb2 values({ts},{i},{i}.0)") - tdSql.query("select apercentile(tb1.c1,100), apercentile(tb2.c2,100) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.query(f"select apercentile(tb1.c1,100), apercentile(tb2.c2,100) from db.tb1 tb1, db.tb2 tb2 where tb1.ts=tb2.ts") tdSql.checkRows(1) tdSql.checkData(0,0,9.000000000) tdSql.checkData(0,0,9.000000000) - # group by - tdSql.execute(" use testdb ") - tdSql.query(" select max(c1),c1 from stb1 group by t1 ") + # group by + tdSql.execute(f"use {dbname} ") + tdSql.query(f" select max(c1),c1 from {dbname}.stb1 group by t1 ") tdSql.checkRows(20) - tdSql.query(" select max(c1),c1 from stb1 group by c1 ") + tdSql.query(f" select max(c1),c1 from {dbname}.stb1 group by c1 ") tdSql.checkRows(30) - tdSql.query(" select max(c1),c2 from stb1 group by c2 ") + tdSql.query(f" select max(c1),c2 from {dbname}.stb1 group by c2 ") tdSql.checkRows(31) # partition by tbname or partition by tag - tdSql.query("select apercentile(c1 ,10)from stb1 partition by tbname") + tdSql.query(f"select apercentile(c1 ,10)from {dbname}.stb1 partition by tbname") query_data = tdSql.queryResult # nest query for support max - tdSql.query("select apercentile(c2+2,10)+1 from (select max(c1) c2 from stb1)") + tdSql.query(f"select apercentile(c2+2,10)+1 from (select max(c1) c2 from {dbname}.stb1)") tdSql.checkData(0,0,31.000000000) - tdSql.query("select apercentile(c1+2,10)+1 as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.query(f"select apercentile(c1+2,10)+1 as c2 from (select ts ,c1 ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,7.560701700) - tdSql.query("select apercentile(a+2,10)+1 as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.query(f"select apercentile(a+2,10)+1 as c2 from (select ts ,abs(c1) a ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,7.560701700) # mixup with other functions - tdSql.query("select max(c1),count(c1),last(c2,c3),spread(c1), apercentile(c1,10) from stb1") + tdSql.query(f"select max(c1),count(c1),last(c2,c3),spread(c1), apercentile(c1,10) from {dbname}.stb1") tdSql.checkData(0,0,28) tdSql.checkData(0,1,184) tdSql.checkData(0,2,-99999) @@ -189,7 +162,7 @@ class TDTestCase: self.check_distribute_datas() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_avg.py b/tests/system-test/2-query/distribute_agg_avg.py index d23a597e92..1b89bb1ff0 100644 --- a/tests/system-test/2-query/distribute_agg_avg.py +++ b/tests/system-test/2-query/distribute_agg_avg.py @@ -7,10 +7,6 @@ import platform class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, - "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -24,7 +20,7 @@ class TDTestCase: avg_sql = f"select avg({col_name}) from {tbname};" same_sql = f"select {col_name} from {tbname} where {col_name} is not null " - + tdSql.query(same_sql) pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): @@ -34,89 +30,66 @@ class TDTestCase: tdSql.query(avg_sql) tdSql.checkData(0,0,pre_avg) - def prepare_datas_of_distribute(self): - + def prepare_datas_of_distribute(self, dbname="testdb"): + # prepate datas for 20 tables distributed at different vgroups - tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") - tdSql.execute(" use testdb ") + tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") + tdSql.execute(f" use {dbname} ") tdSql.execute( - '''create table stb1 + f'''create table {dbname}.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) ''' ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) for i in range(20): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) for i in range(1,21): if i ==1 or i == 4: continue else: - tbname = "ct"+f'{i}' + tbname = f"{dbname}.ct{i}" for j in range(9): tdSql.execute( f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - - tdSql.execute( - f'''insert into t1 values - ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) - ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) - ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) - ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) - ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) - ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) - ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) - ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) - ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) - ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ''' - ) + tdSql.execute(f"insert into {dbname}.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdLog.info(" prepare data for distributed_aggregate done! ") - def check_distribute_datas(self): + def check_distribute_datas(self, dbname="testdb"): # get vgroup_ids of all - tdSql.query("show vgroups ") + tdSql.query(f"show {dbname}.vgroups ") vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed - tdSql.query("show tables like 'ct%'") + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -126,15 +99,15 @@ class TDTestCase: if count < 2: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") - def check_avg_distribute_diff_vnode(self,col_name): - + def check_avg_distribute_diff_vnode(self,col_name, dbname="testdb"): + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -143,10 +116,10 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - - avg_sql = f"select avg({col_name}) from stb1 where tbname in ({tbname_filters});" - same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null " + avg_sql = f"select avg({col_name}) from {dbname}.stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from {dbname}.stb1 where tbname in ({tbname_filters}) and {col_name} is not null " tdSql.query(same_sql) pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] @@ -157,105 +130,105 @@ class TDTestCase: tdSql.query(avg_sql) tdSql.checkData(0,0,pre_avg) - def check_avg_status(self): - # check max function work status - - tdSql.query("show tables like 'ct%'") + def check_avg_status(self, dbname="testdb"): + # check max function work status + + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - tablenames.append(table_name[0]) + tablenames.append(f"{dbname}.{table_name[0]}") - tdSql.query("desc stb1") + tdSql.query(f"desc {dbname}.stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_avg_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): - self.check_avg_distribute_diff_vnode(colname) + self.check_avg_distribute_diff_vnode(colname, dbname) else: - # self.check_avg_distribute_diff_vnode(colname) # bug for tag + # self.check_avg_distribute_diff_vnode(colname, dbname) # bug for tag pass - - def distribute_agg_query(self): + + def distribute_agg_query(self, dbname="testdb"): # basic filter - tdSql.query(" select avg(c1) from stb1 ") + tdSql.query(f"select avg(c1) from {dbname}.stb1 ") tdSql.checkData(0,0,14.086956522) - tdSql.query(" select avg(a) from (select avg(c1) a from stb1 partition by tbname) ") + tdSql.query(f"select avg(a) from (select avg(c1) a from {dbname}.stb1 partition by tbname) ") tdSql.checkData(0,0,14.292307692) - tdSql.query(" select avg(c1) from stb1 where t1=1") + tdSql.query(f"select avg(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,6.000000000) - tdSql.query("select avg(c1+c2) from stb1 where c1 =1 ") + tdSql.query(f"select avg(c1+c2) from {dbname}.stb1 where c1 =1 ") tdSql.checkData(0,0,11112.000000000) - tdSql.query("select avg(c1) from stb1 where tbname=\"ct2\"") + tdSql.query(f"select avg(c1) from {dbname}.stb1 where tbname=\"ct2\"") tdSql.checkData(0,0,6.000000000) - tdSql.query("select avg(c1) from stb1 partition by tbname") + tdSql.query(f"select avg(c1) from {dbname}.stb1 partition by tbname") tdSql.checkRows(20) - tdSql.query("select avg(c1) from stb1 where t1> 4 partition by tbname") + tdSql.query(f"select avg(c1) from {dbname}.stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all - tdSql.query("select avg(c1) from stb1 union all select avg(c1) from stb1 ") + # union all + tdSql.query(f"select avg(c1) from {dbname}.stb1 union all select avg(c1) from {dbname}.stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,14.086956522) - tdSql.query("select avg(a) from (select avg(c1) a from stb1 union all select avg(c1) a from stb1)") + tdSql.query(f"select avg(a) from (select avg(c1) a from {dbname}.stb1 union all select avg(c1) a from {dbname}.stb1)") tdSql.checkRows(1) tdSql.checkData(0,0,14.086956522) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") - tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") - tdSql.execute(" create table tb1 using st tags(1) ") - tdSql.execute(" create table tb2 using st tags(2) ") + tdSql.execute(" create stable db.st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table db.tb1 using db.st tags(1) ") + tdSql.execute(" create table db.tb2 using db.st tags(2) ") + - for i in range(10): ts = i*10 + self.ts - tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") - tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb2 values({ts},{i},{i}.0)") - tdSql.query("select avg(tb1.c1), avg(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.query(f"select avg(tb1.c1), avg(tb2.c2) from db.tb1 tb1, db.tb2 tb2 where tb1.ts=tb2.ts") tdSql.checkRows(1) tdSql.checkData(0,0,4.500000000) tdSql.checkData(0,1,4.500000000) - # group by - tdSql.execute(" use testdb ") + # group by + tdSql.execute(f" use {dbname} ") # partition by tbname or partition by tag - tdSql.query("select avg(c1) from stb1 partition by tbname") + tdSql.query(f"select avg(c1) from {dbname}.stb1 partition by tbname") tdSql.checkRows(20) # nest query for support max - tdSql.query("select avg(c2+2)+1 from (select avg(c1) c2 from stb1)") + tdSql.query(f"select avg(c2+2)+1 from (select avg(c1) c2 from {dbname}.stb1)") tdSql.checkData(0,0,17.086956522) - tdSql.query("select avg(c1+2) as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.query(f"select avg(c1+2) as c2 from (select ts ,c1 ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,16.086956522) - tdSql.query("select avg(a+2) as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.query(f"select avg(a+2) as c2 from (select ts ,abs(c1) a ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,16.086956522) # mixup with other functions - tdSql.query("select max(c1),count(c1),last(c2,c3),sum(c1+c2),avg(c1) from stb1") + tdSql.query(f"select max(c1),count(c1),last(c2,c3),sum(c1+c2),avg(c1) from {dbname}.stb1") tdSql.checkData(0,0,28) tdSql.checkData(0,1,184) tdSql.checkData(0,2,-99999) @@ -270,7 +243,7 @@ class TDTestCase: self.check_avg_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index 6b7e7375ff..6f8183bf06 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -51,32 +51,32 @@ class TDTestCase: tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) # tdLog.info("insert data") # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) - + tdLog.info("create 4 topics") sqlString = "create topic %s as database %s" %(topicNameList[0], paraDict['dbName']) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) - + sqlString = "create topic %s as stable %s.%s" %(topicNameList[1], paraDict['dbName'], paraDict['stbName']) tdLog.info("create topic sql: %s"%sqlString) - tdSql.execute(sqlString) + tdSql.execute(sqlString) queryString = "select * from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s" %(topicNameList[2], queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) - + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) sqlString = "create topic %s as %s " %(topicNameList[3], queryString) tdLog.info("create topic sql: %s"%sqlString) tdSql.execute(sqlString) tdSql.query("show topics") - tdLog.debug(tdSql.queryResult) + tdLog.debug(tdSql.queryResult) rows = tdSql.getRows() if rows != len(consumerIdList): tdLog.exit("topic rows error") - + for i in range (rows): topicName = tdSql.getData(i,0) matchFlag = 0 @@ -87,24 +87,24 @@ class TDTestCase: break if matchFlag == 0: tdLog.exit("topic name: %s is error", topicName) - + # init consume info, and start tmq_sim, then check consume result tdLog.info("insert consume info to consume processor") expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] topicList = topicNameList[0] ifcheckdata = 0 - ifManualCommit = 0 + ifManualCommit = 0 keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[0] tmqCom.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - + topicList = topicNameList[1] keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[1] tmqCom.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - + topicList = topicNameList[2] keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[2] tmqCom.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) - + topicList = topicNameList[3] keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[3] tmqCom.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) @@ -118,27 +118,27 @@ class TDTestCase: time.sleep(5) tdLog.info("check show consumers") tdSql.query("show consumers") - # tdLog.info(tdSql.queryResult) + # tdLog.info(tdSql.queryResult) rows = tdSql.getRows() tdLog.info("show consumers rows: %d"%rows) if rows != len(topicNameList): tdLog.exit("show consumers rows error") - - tdLog.info("check show subscriptions") + + tdLog.info("check show subscriptions") tdSql.query("show subscriptions") - # tdLog.debug(tdSql.queryResult) + # tdLog.debug(tdSql.queryResult) rows = tdSql.getRows() tdLog.info("show subscriptions rows: %d"%rows) if rows != paraDict['vgroups'] * len(topicNameList): tdLog.exit("show subscriptions rows error") pThread.join() - + tdLog.info("insert process end, and start to check consume result") expectRows = len(consumerIdList) _ = tmqCom.selectConsumeResult(expectRows) - - time.sleep(10) + + time.sleep(10) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4588474753..d2cbba19b2 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -58,14 +58,36 @@ python3 ./test.py -f 2-query/char_length.py python3 ./test.py -f 2-query/char_length.py -R python3 ./test.py -f 2-query/check_tsdb.py python3 ./test.py -f 2-query/check_tsdb.py -R +python3 ./test.py -f 2-query/concat.py +python3 ./test.py -f 2-query/concat.py -R +python3 ./test.py -f 2-query/concat_ws.py +python3 ./test.py -f 2-query/concat_ws.py -R +python3 ./test.py -f 2-query/concat_ws2.py +python3 ./test.py -f 2-query/concat_ws2.py -R +python3 ./test.py -f 2-query/cos.py +python3 ./test.py -f 2-query/cos.py -R +python3 ./test.py -f 2-query/count_partition.py +python3 ./test.py -f 2-query/count_partition.py -R +python3 ./test.py -f 2-query/count.py +python3 ./test.py -f 2-query/count.py -R +python3 ./test.py -f 2-query/db.py +python3 ./test.py -f 2-query/db.py -R +python3 ./test.py -f 2-query/diff.py +python3 ./test.py -f 2-query/diff.py -R +python3 ./test.py -f 2-query/distinct.py +python3 ./test.py -f 2-query/distinct.py -R +python3 ./test.py -f 2-query/distribute_agg_apercentile.py +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R +python3 ./test.py -f 2-query/distribute_agg_avg.py +python3 ./test.py -f 2-query/distribute_agg_avg.py -R + + + + python3 ./test.py -f 1-insert/update_data.py python3 ./test.py -f 1-insert/delete_data.py -python3 ./test.py -f 2-query/db.py - -python3 ./test.py -f 2-query/db.py -python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py python3 ./test.py -f 2-query/ltrim.py python3 ./test.py -f 2-query/rtrim.py @@ -77,10 +99,7 @@ python3 ./test.py -f 2-query/join2.py python3 ./test.py -f 2-query/substr.py python3 ./test.py -f 2-query/union.py python3 ./test.py -f 2-query/union1.py -python3 ./test.py -f 2-query/concat.py python3 ./test.py -f 2-query/concat2.py -python3 ./test.py -f 2-query/concat_ws.py -python3 ./test.py -f 2-query/concat_ws2.py python3 ./test.py -f 2-query/spread.py python3 ./test.py -f 2-query/hyperloglog.py python3 ./test.py -f 2-query/explain.py @@ -93,13 +112,11 @@ python3 ./test.py -f 2-query/Now.py python3 ./test.py -f 2-query/Today.py python3 ./test.py -f 2-query/max.py python3 ./test.py -f 2-query/min.py -python3 ./test.py -f 2-query/count.py python3 ./test.py -f 2-query/last.py python3 ./test.py -f 2-query/first.py python3 ./test.py -f 2-query/To_iso8601.py python3 ./test.py -f 2-query/To_unixtimestamp.py python3 ./test.py -f 2-query/timetruncate.py -python3 ./test.py -f 2-query/diff.py python3 ./test.py -f 2-query/Timediff.py python3 ./test.py -f 2-query/json_tag.py @@ -111,7 +128,6 @@ python3 ./test.py -f 2-query/log.py python3 ./test.py -f 2-query/pow.py python3 ./test.py -f 2-query/sqrt.py python3 ./test.py -f 2-query/sin.py -python3 ./test.py -f 2-query/cos.py python3 ./test.py -f 2-query/tan.py python3 ./test.py -f 2-query/query_cols_tags_and_or.py # python3 ./test.py -f 2-query/nestedQuery.py @@ -122,7 +138,6 @@ python3 ./test.py -f 2-query/query_cols_tags_and_or.py python3 ./test.py -f 2-query/elapsed.py python3 ./test.py -f 2-query/csum.py python3 ./test.py -f 2-query/mavg.py -python3 ./test.py -f 2-query/diff.py python3 ./test.py -f 2-query/sample.py python3 ./test.py -f 2-query/function_diff.py python3 ./test.py -f 2-query/unique.py @@ -136,12 +151,9 @@ python3 ./test.py -f 2-query/distribute_agg_max.py python3 ./test.py -f 2-query/distribute_agg_min.py python3 ./test.py -f 2-query/distribute_agg_sum.py python3 ./test.py -f 2-query/distribute_agg_spread.py -python3 ./test.py -f 2-query/distribute_agg_apercentile.py -python3 ./test.py -f 2-query/distribute_agg_avg.py python3 ./test.py -f 2-query/distribute_agg_stddev.py python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 2-query/irate.py -python3 ./test.py -f 2-query/count_partition.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 2-query/max_partition.py @@ -162,9 +174,9 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 # python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 # BUG Redict python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 # python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 5 -M 3 - python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 + python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 6 -M 3 -C 5 + - python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb0.py @@ -281,7 +293,6 @@ python3 ./test.py -f 2-query/avg.py -Q 2 # python3 ./test.py -f 2-query/elapsed.py -Q 2 python3 ./test.py -f 2-query/csum.py -Q 2 python3 ./test.py -f 2-query/mavg.py -Q 2 -python3 ./test.py -f 2-query/diff.py -Q 2 python3 ./test.py -f 2-query/sample.py -Q 2 python3 ./test.py -f 2-query/function_diff.py -Q 2 python3 ./test.py -f 2-query/unique.py -Q 2 @@ -369,7 +380,6 @@ python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 # python3 ./test.py -f 2-query/elapsed.py -Q 3 python3 ./test.py -f 2-query/csum.py -Q 3 python3 ./test.py -f 2-query/mavg.py -Q 3 -python3 ./test.py -f 2-query/diff.py -Q 3 python3 ./test.py -f 2-query/sample.py -Q 3 python3 ./test.py -f 2-query/function_diff.py -Q 3 python3 ./test.py -f 2-query/unique.py -Q 3 From cd0e66a694c5a5315847349f4e51a73b197b5d89 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 22 Jul 2022 09:45:56 +0800 Subject: [PATCH 02/43] fix case --- .../2-query/distribute_agg_apercentile.py | 1 + .../system-test/2-query/distribute_agg_avg.py | 1 + .../2-query/distribute_agg_count.py | 186 +++++++--------- .../system-test/2-query/distribute_agg_max.py | 195 ++++++++--------- .../system-test/2-query/distribute_agg_min.py | 201 ++++++++---------- tests/system-test/fulltest.sh | 9 +- 6 files changed, 257 insertions(+), 336 deletions(-) diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index 591824fdd3..1fd853f9eb 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -7,6 +7,7 @@ import random class TDTestCase: + updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/distribute_agg_avg.py b/tests/system-test/2-query/distribute_agg_avg.py index 1b89bb1ff0..3892ae0da1 100644 --- a/tests/system-test/2-query/distribute_agg_avg.py +++ b/tests/system-test/2-query/distribute_agg_avg.py @@ -8,6 +8,7 @@ import platform class TDTestCase: + updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index ebca81545c..835d1eeb57 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -2,15 +2,12 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, - "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) @@ -25,7 +22,7 @@ class TDTestCase: same_sql = f"select sum(c) from (select {col_name} ,1 as c from {tbname} where {col_name} is not null) " tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -35,90 +32,66 @@ class TDTestCase: else: tdLog.info(" count function work as expected, sql : %s "% max_sql) + def prepare_datas_of_distribute(self, dbname="testdb"): - def prepare_datas_of_distribute(self): - # prepate datas for 20 tables distributed at different vgroups - tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") - tdSql.execute(" use testdb ") + tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") + tdSql.execute(f" use {dbname} ") tdSql.execute( - '''create table stb1 + f'''create table {dbname}.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) ''' ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) for i in range(20): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) for i in range(1,21): if i ==1 or i == 4: continue else: - tbname = "ct"+f'{i}' + tbname = f"{dbname}.ct{i}" for j in range(9): tdSql.execute( f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - - tdSql.execute( - f'''insert into t1 values - ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) - ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) - ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) - ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) - ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) - ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) - ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) - ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) - ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) - ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ''' - ) + tdSql.execute(f"insert into {dbname}.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdLog.info(" prepare data for distributed_aggregate done! ") - def check_distribute_datas(self): + def check_distribute_datas(self, dbname="testdb"): # get vgroup_ids of all - tdSql.query("show vgroups ") + tdSql.query(f"show {dbname}.vgroups ") vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed - tdSql.query("show tables like 'ct%'") + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -128,15 +101,15 @@ class TDTestCase: if count < 2: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") - def check_count_distribute_diff_vnode(self,col_name): - + def check_count_distribute_diff_vnode(self,col_name, dbname="testdb"): + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -145,13 +118,13 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - - max_sql = f"select count({col_name}) from stb1 where tbname in ({tbname_filters});" - same_sql = f"select sum(c) from (select {col_name} ,1 as c from stb1 where tbname in ({tbname_filters}) and {col_name} is not null) " + max_sql = f"select count({col_name}) from {dbname}.stb1 where tbname in ({tbname_filters});" + + same_sql = f"select sum(c) from (select {col_name} ,1 as c from {dbname}.stb1 where tbname in ({tbname_filters}) and {col_name} is not null) " tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -161,120 +134,119 @@ class TDTestCase: else: tdLog.info(" count function work as expected, sql : %s "% max_sql) - def check_count_status(self): - # check max function work status - - tdSql.query("show tables like 'ct%'") + def check_count_status(self, dbname="testdb"): + # check max function work status + + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - tablenames.append(table_name[0]) + tablenames.append(f"{dbname}.{table_name[0]}") - tdSql.query("desc stb1") + tdSql.query(f"desc {dbname}.stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_count_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): - self.check_count_distribute_diff_vnode(colname) + self.check_count_distribute_diff_vnode(colname, dbname) else: - # self.check_count_distribute_diff_vnode(colname) # bug for tag + # self.check_count_distribute_diff_vnode(colname, dbname) # bug for tag pass - - def distribute_agg_query(self): + def distribute_agg_query(self, dbname="testdb"): # basic filter - tdSql.query("select count(c1) from stb1 ") + tdSql.query(f"select count(c1) from {dbname}.stb1 ") tdSql.checkData(0,0,184) - tdSql.query("select count(c1) from stb1 where t1=1") + tdSql.query(f"select count(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,9) - tdSql.query("select count(c1+c2) from stb1 where c1 =1 ") + tdSql.query(f"select count(c1+c2) from {dbname}.stb1 where c1 =1 ") tdSql.checkData(0,0,2) - tdSql.query("select count(c1) from stb1 where tbname=\"ct2\"") + tdSql.query(f"select count(c1) from {dbname}.stb1 where tbname=\"ct2\"") tdSql.checkData(0,0,9) - tdSql.query("select count(c1) from stb1 partition by tbname") + tdSql.query(f"select count(c1) from {dbname}.stb1 partition by tbname") tdSql.checkRows(20) - tdSql.query("select count(c1) from stb1 where t1> 4 partition by tbname") + tdSql.query(f"select count(c1) from {dbname}.stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all - tdSql.query("select count(c1) from stb1 union all select count(c1) from stb1 ") + # union all + tdSql.query(f"select count(c1) from {dbname}.stb1 union all select count(c1) from {dbname}.stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,184) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") - tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") - tdSql.execute(" create table tb1 using st tags(1) ") - tdSql.execute(" create table tb2 using st tags(2) ") + tdSql.execute(" create stable db.st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table db.tb1 using db.st tags(1) ") + tdSql.execute(" create table db.tb2 using db.st tags(2) ") + - for i in range(10): ts = i*10 + self.ts - tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") - tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb2 values({ts},{i},{i}.0)") - tdSql.query("select count(tb1.c1), count(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.query(f"select count(tb1.c1), count(tb2.c2) from db.tb1 tb1, db.tb2 tb2 where tb1.ts=tb2.ts") tdSql.checkRows(1) tdSql.checkData(0,0,10) tdSql.checkData(0,1,10) - # group by - tdSql.execute(" use testdb ") + # group by + tdSql.execute(f" use {dbname} ") - tdSql.query(" select count(*) from stb1 ") + tdSql.query(f"select count(*) from {dbname}.stb1 ") tdSql.checkData(0,0,187) - tdSql.query(" select count(*) from stb1 group by t1 ") + tdSql.query(f"select count(*) from {dbname}.stb1 group by t1 ") tdSql.checkRows(20) - tdSql.query(" select count(*) from stb1 group by c1 ") + tdSql.query(f"select count(*) from {dbname}.stb1 group by c1 ") tdSql.checkRows(30) - tdSql.query(" select count(*) from stb1 group by c2 ") + tdSql.query(f"select count(*) from {dbname}.stb1 group by c2 ") tdSql.checkRows(31) # partition by tbname or partition by tag - tdSql.query("select max(c1),tbname from stb1 partition by tbname") + tdSql.query(f"select max(c1),tbname from {dbname}.stb1 partition by tbname") query_data = tdSql.queryResult - + for row in query_data: - tbname = row[1] - tdSql.query(" select max(c1) from %s "%tbname) + tbname = f"{dbname}.{row[1]}" + tdSql.query(f"select max(c1) from %s "%tbname) tdSql.checkData(0,0,row[0]) - tdSql.query("select max(c1),tbname from stb1 partition by t1") + tdSql.query(f"select max(c1),tbname from {dbname}.stb1 partition by t1") query_data = tdSql.queryResult - + for row in query_data: - tbname = row[1] - tdSql.query(" select max(c1) from %s "%tbname) + tbname = f"{dbname}.{row[1]}" + tdSql.query(f"select max(c1) from %s "%tbname) tdSql.checkData(0,0,row[0]) # nest query for support max - tdSql.query("select abs(c2+2)+1 from (select count(c1) c2 from stb1)") + tdSql.query(f"select abs(c2+2)+1 from (select count(c1) c2 from {dbname}.stb1)") tdSql.checkData(0,0,187.000000000) - tdSql.query("select count(c1+2) as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.query(f"select count(c1+2) as c2 from (select ts ,c1 ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,184) - tdSql.query("select count(a+2) as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.query(f"select count(a+2) as c2 from (select ts ,abs(c1) a ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,184) # mixup with other functions - tdSql.query("select max(c1),count(c1),last(c2,c3) from stb1") + tdSql.query(f"select max(c1),count(c1),last(c2,c3) from {dbname}.stb1") tdSql.checkData(0,0,28) tdSql.checkData(0,1,184) tdSql.checkData(0,2,-99999) @@ -287,7 +259,7 @@ class TDTestCase: self.check_count_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index c7e074095b..a7b31a2084 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -2,14 +2,12 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, - "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -25,7 +23,7 @@ class TDTestCase: same_sql = f"select {col_name} from {tbname} order by {col_name} desc limit 1" tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -36,89 +34,65 @@ class TDTestCase: tdLog.info(" max function work as expected, sql : %s "% max_sql) - def prepare_datas_of_distribute(self): - + def prepare_datas_of_distribute(self, dbname="testdb"): + # prepate datas for 20 tables distributed at different vgroups - tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") - tdSql.execute(" use testdb ") + tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") + tdSql.execute(f" use {dbname} ") tdSql.execute( - '''create table stb1 + f'''create table {dbname}.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) ''' ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) for i in range(20): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) for i in range(1,21): if i ==1 or i == 4: continue else: - tbname = "ct"+f'{i}' + tbname = f"{dbname}.ct{i}" for j in range(9): tdSql.execute( f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - - tdSql.execute( - f'''insert into t1 values - ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) - ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) - ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) - ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) - ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) - ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) - ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) - ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) - ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) - ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ''' - ) + tdSql.execute(f"insert into {dbname}.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdLog.info(" prepare data for distributed_aggregate done! ") - def check_distribute_datas(self): + def check_distribute_datas(self, dbname="testdb"): # get vgroup_ids of all - tdSql.query("show vgroups ") + tdSql.query(f"show {dbname}.vgroups ") vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - # check sub_table of per vnode ,make sure sub_table has been distributed - tdSql.query("show tables like 'ct%'") + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -128,15 +102,15 @@ class TDTestCase: if count < 2: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") - def check_max_distribute_diff_vnode(self,col_name): - + def check_max_distribute_diff_vnode(self,col_name, dbname="testdb"): + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -145,13 +119,13 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - - max_sql = f"select max({col_name}) from stb1 where tbname in ({tbname_filters});" - same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) order by {col_name} desc limit 1" + max_sql = f"select max({col_name}) from {dbname}.stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from {dbname}.stb1 where tbname in ({tbname_filters}) order by {col_name} desc limit 1" tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -161,138 +135,137 @@ class TDTestCase: else: tdLog.info(" max function work as expected, sql : %s "% max_sql) - def check_max_status(self): - # check max function work status - - tdSql.query("show tables like 'ct%'") + def check_max_status(self, dbname="testdb"): + # check max function work status + + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - tablenames.append(table_name[0]) + tablenames.append(f"{dbname}.{table_name[0]}") - tdSql.query("desc stb1") + tdSql.query(f"desc {dbname}.stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_max_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): - self.check_max_distribute_diff_vnode(colname) + self.check_max_distribute_diff_vnode(colname, dbname) else: - # self.check_max_distribute_diff_vnode(colname) # bug for tag + # self.check_max_distribute_diff_vnode(colname, dbname) # bug for tag pass - - def distribute_agg_query(self): + def distribute_agg_query(self, dbname="testdb"): # basic filter - tdSql.query("select max(c1) from stb1 where c1 is null") + tdSql.query(f"select max(c1) from {dbname}.stb1 where c1 is null") tdSql.checkRows(0) - tdSql.query("select max(c1) from stb1 where t1=1") + tdSql.query(f"select max(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,10) - tdSql.query("select max(c1+c2) from stb1 where c1 =1 ") + tdSql.query(f"select max(c1+c2) from {dbname}.stb1 where c1 =1 ") tdSql.checkData(0,0,11112.000000000) - tdSql.query("select max(c1) from stb1 where tbname=\"ct2\"") + tdSql.query(f"select max(c1) from {dbname}.stb1 where tbname=\"ct2\"") tdSql.checkData(0,0,10) - tdSql.query("select max(c1) from stb1 partition by tbname") + tdSql.query(f"select max(c1) from {dbname}.stb1 partition by tbname") tdSql.checkRows(20) - tdSql.query("select max(c1) from stb1 where t1> 4 partition by tbname") + tdSql.query(f"select max(c1) from {dbname}.stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all - tdSql.query("select max(c1) from stb1 union all select max(c1) from stb1 ") + # union all + tdSql.query(f"select max(c1) from {dbname}.stb1 union all select max(c1) from {dbname}.stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,28) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") - tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") - tdSql.execute(" create table tb1 using st tags(1) ") - tdSql.execute(" create table tb2 using st tags(2) ") + tdSql.execute(" create stable db.st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table db.tb1 using db.st tags(1) ") + tdSql.execute(" create table db.tb2 using db.st tags(2) ") + - for i in range(10): ts = i*10 + self.ts - tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") - tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb2 values({ts},{i},{i}.0)") - tdSql.query("select max(tb1.c1), tb2.c2 from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.query(f"select max(tb1.c1), tb2.c2 from db.tb1 tb1, db.tb2 tb2 where tb1.ts=tb2.ts") tdSql.checkRows(1) tdSql.checkData(0,0,9) tdSql.checkData(0,0,9.00000) - # group by - tdSql.execute(" use testdb ") - tdSql.query(" select max(c1),c1 from stb1 group by t1 ") + # group by + tdSql.execute("use testdb ") + tdSql.query(f"select max(c1),c1 from {dbname}.stb1 group by t1 ") tdSql.checkRows(20) - tdSql.query(" select max(c1),c1 from stb1 group by c1 ") + tdSql.query(f"select max(c1),c1 from {dbname}.stb1 group by c1 ") tdSql.checkRows(30) - tdSql.query(" select max(c1),c2 from stb1 group by c2 ") + tdSql.query(f"select max(c1),c2 from {dbname}.stb1 group by c2 ") tdSql.checkRows(31) # selective common cols of datas - tdSql.query("select max(c1),c2,c3,c5 from stb1") + tdSql.query(f"select max(c1),c2,c3,c5 from {dbname}.stb1") tdSql.checkRows(1) tdSql.checkData(0,0,28) tdSql.checkData(0,1,311108) tdSql.checkData(0,2,3108) tdSql.checkData(0,3,31.08000) - tdSql.query("select max(c1),t1,c2,t3 from stb1") + tdSql.query(f"select max(c1),t1,c2,t3 from {dbname}.stb1") tdSql.checkRows(1) tdSql.checkData(0,0,28) tdSql.checkData(0,1,19) tdSql.checkData(0,2,311108) - tdSql.query("select max(c1),ceil(t1),pow(c2,1)+2,abs(t3) from stb1") + tdSql.query(f"select max(c1),ceil(t1),pow(c2,1)+2,abs(t3) from {dbname}.stb1") tdSql.checkRows(1) tdSql.checkData(0,0,28) tdSql.checkData(0,1,19) - tdSql.checkData(0,2,311110.000000000) - tdSql.checkData(0,3,2109) + tdSql.checkData(0,2,311110.000000000) + tdSql.checkData(0,3,2109) # partition by tbname or partition by tag - tdSql.query("select max(c1),tbname from stb1 partition by tbname") + tdSql.query(f"select max(c1),tbname from {dbname}.stb1 partition by tbname") query_data = tdSql.queryResult - + for row in query_data: - tbname = row[1] - tdSql.query(" select max(c1) from %s "%tbname) + tbname = f"{dbname}.{row[1]}" + tdSql.query(f"select max(c1) from %s "%tbname) tdSql.checkData(0,0,row[0]) - tdSql.query("select max(c1),tbname from stb1 partition by t1") + tdSql.query(f"select max(c1),tbname from {dbname}.stb1 partition by t1") query_data = tdSql.queryResult - + for row in query_data: - tbname = row[1] - tdSql.query(" select max(c1) from %s "%tbname) + tbname = f"{dbname}.{row[1]}" + tdSql.query(f"select max(c1) from %s "%tbname) tdSql.checkData(0,0,row[0]) # nest query for support max - tdSql.query("select abs(c2+2)+1 from (select max(c1) c2 from stb1)") + tdSql.query(f"select abs(c2+2)+1 from (select max(c1) c2 from {dbname}.stb1)") tdSql.checkData(0,0,31.000000000) - tdSql.query("select max(c1+2)+1 as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.query(f"select max(c1+2)+1 as c2 from (select ts ,c1 ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,31.000000000) - tdSql.query("select max(a+2)+1 as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.query(f"select max(a+2)+1 as c2 from (select ts ,abs(c1) a ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,31.000000000) # mixup with other functions - tdSql.query("select max(c1),count(c1),last(c2,c3) from stb1") + tdSql.query(f"select max(c1),count(c1),last(c2,c3) from {dbname}.stb1") tdSql.checkData(0,0,28) tdSql.checkData(0,1,184) tdSql.checkData(0,2,-99999) @@ -305,7 +278,7 @@ class TDTestCase: self.check_max_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index d8f93a01f5..6fea94b61f 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -2,14 +2,12 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, - "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + updatecfgDict = {"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) @@ -25,7 +23,7 @@ class TDTestCase: same_sql = f"select {col_name} from {tbname} where {col_name} is not null order by {col_name} asc limit 1" tdSql.query(min_sql) - min_result = tdSql.queryResult + min_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -35,90 +33,65 @@ class TDTestCase: else: tdLog.info(" min function work as expected, sql : %s "% min_sql) + def prepare_datas_of_distribute(self, dbname="testdb"): - def prepare_datas_of_distribute(self): - # prepate datas for 20 tables distributed at different vgroups - tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") - tdSql.execute(" use testdb ") + tdSql.execute(f"create database if not exists {dbname} keep 3650 duration 1000 vgroups 5") + tdSql.execute(f" use {dbname} ") tdSql.execute( - '''create table stb1 + f'''create table {dbname}.stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) ''' ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) for i in range(20): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + tdSql.execute(f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') for i in range(9): tdSql.execute( - f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) tdSql.execute( - f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + f"insert into {dbname}.ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) for i in range(1,21): if i ==1 or i == 4: continue else: - tbname = "ct"+f'{i}' + tbname = f"{dbname}.ct{i}" for j in range(9): tdSql.execute( f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute(f"insert into {dbname}.ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - - tdSql.execute( - f'''insert into t1 values - ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) - ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) - ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) - ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) - ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) - ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) - ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) - ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) - ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) - ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ''' - ) + tdSql.execute(f"insert into {dbname}.ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute(f"insert into {dbname}.ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdLog.info(" prepare data for distributed_aggregate done! ") - def check_distribute_datas(self): + def check_distribute_datas(self, dbname="testdb"): # get vgroup_ids of all - tdSql.query("show vgroups ") + tdSql.query(f"show {dbname}.vgroups ") vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - # check sub_table of per vnode ,make sure sub_table has been distributed - tdSql.query("show tables like 'ct%'") + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -128,15 +101,15 @@ class TDTestCase: if count < 2: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") - def check_min_distribute_diff_vnode(self,col_name): - + def check_min_distribute_diff_vnode(self,col_name, dbname="testdb"): + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -145,13 +118,13 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - - min_sql = f"select min({col_name}) from stb1 where tbname in ({tbname_filters});" - same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null order by {col_name} asc limit 1" + min_sql = f"select min({col_name}) from {dbname}.stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from {dbname}.stb1 where tbname in ({tbname_filters}) and {col_name} is not null order by {col_name} asc limit 1" tdSql.query(min_sql) - min_result = tdSql.queryResult + min_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -161,140 +134,138 @@ class TDTestCase: else: tdLog.info(" min function work as expected, sql : %s "% min_sql) - def check_min_status(self): - # check max function work status - - tdSql.query("show tables like 'ct%'") + def check_min_status(self, dbname="testdb"): + # check min function work status + + tdSql.query(f"show {dbname}.tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - tablenames.append(table_name[0]) + tablenames.append(f"{dbname}.{table_name[0]}") - tdSql.query("desc stb1") + tdSql.query(f"desc {dbname}.stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_min_functions(tablename,colname) - # check max function for different vnode + # check min function for different vnode for colname in colnames: if colname.startswith("c"): - self.check_min_distribute_diff_vnode(colname) + self.check_min_distribute_diff_vnode(colname, dbname) else: - # self.check_min_distribute_diff_vnode(colname) # bug for tag + # self.check_min_distribute_diff_vnode(colname, dbname) # bug for tag pass - - def distribute_agg_query(self): + def distribute_agg_query(self, dbname="testdb"): # basic filter - tdSql.query("select min(c1) from stb1 where c1 is null") + tdSql.query(f"select min(c1) from {dbname}.stb1 where c1 is null") tdSql.checkRows(0) - tdSql.query("select min(c1) from stb1 where t1=1") + tdSql.query(f"select min(c1) from {dbname}.stb1 where t1=1") tdSql.checkData(0,0,2) - tdSql.query("select min(c1+c2) from stb1 where c1 =1 ") + tdSql.query(f"select min(c1+c2) from {dbname}.stb1 where c1 =1 ") tdSql.checkData(0,0,11112.000000000) - tdSql.query("select min(c1) from stb1 where tbname=\"ct2\"") - tdSql.checkData(0,0,2) + tdSql.query(f"select min(c1) from {dbname}.stb1 where tbname=\"ct2\"") + tdSql.checkData(0, 0, 2) - tdSql.query("select min(c1) from stb1 partition by tbname") + tdSql.query(f"select min(c1) from {dbname}.stb1 partition by tbname") tdSql.checkRows(20) - tdSql.query("select min(c1) from stb1 where t1> 4 partition by tbname") + tdSql.query(f"select min(c1) from {dbname}.stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all - tdSql.query("select min(c1) from stb1 union all select min(c1) from stb1 ") + # union all + tdSql.query(f"select min(c1) from {dbname}.stb1 union all select min(c1) from {dbname}.stb1 ") tdSql.checkRows(2) - tdSql.checkData(0,0,0) + tdSql.checkData(0, 0, 0) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") - tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") - tdSql.execute(" create table tb1 using st tags(1) ") - tdSql.execute(" create table tb2 using st tags(2) ") + tdSql.execute(" create stable db.st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table db.tb1 using db.st tags(1) ") + tdSql.execute(" create table db.tb2 using db.st tags(2) ") + - for i in range(10): ts = i*10 + self.ts - tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") - tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into db.tb2 values({ts},{i},{i}.0)") - tdSql.query("select min(tb1.c1), tb2.c2 from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.query(f"select min(tb1.c1), tb2.c2 from db.tb1 tb1, db.tb2 tb2 where tb1.ts=tb2.ts") tdSql.checkRows(1) tdSql.checkData(0,0,0) tdSql.checkData(0,0,0.00000) - # group by - tdSql.execute(" use testdb ") - tdSql.query(" select min(c1),c1 from stb1 group by t1 ") + # group by + tdSql.execute("use testdb ") + tdSql.query(f"select min(c1),c1 from {dbname}.stb1 group by t1 ") tdSql.checkRows(20) - tdSql.query(" select min(c1),c1 from stb1 group by c1 ") + tdSql.query(f"select min(c1),c1 from {dbname}.stb1 group by c1 ") tdSql.checkRows(30) - tdSql.query(" select min(c1),c2 from stb1 group by c2 ") + tdSql.query(f"select min(c1),c2 from {dbname}.stb1 group by c2 ") tdSql.checkRows(31) # selective common cols of datas - tdSql.query("select min(c1),c2,c3,c5 from stb1") + tdSql.query(f"select min(c1),c2,c3,c5 from {dbname}.stb1") tdSql.checkRows(1) tdSql.checkData(0,0,0) tdSql.checkData(0,1,0) tdSql.checkData(0,2,0) tdSql.checkData(0,3,0) - tdSql.query("select min(c1),t1,c2,t3 from stb1 where c1 >5") + tdSql.query(f"select min(c1),t1,c2,t3 from {dbname}.stb1 where c1 > 5") tdSql.checkRows(1) tdSql.checkData(0,0,6) tdSql.checkData(0,2,66666) - tdSql.query("select min(c1),ceil(t1),pow(c2,1)+2,abs(t3) from stb1 where c1>12") + tdSql.query(f"select min(c1),ceil(t1),pow(c2,1)+2,abs(t3) from {dbname}.stb1 where c1 > 12") tdSql.checkRows(1) tdSql.checkData(0,0,13) - tdSql.checkData(0,2,144445.000000000) - + tdSql.checkData(0,2,144445.000000000) + # partition by tbname or partition by tag - tdSql.query("select min(c1),tbname from stb1 partition by tbname") + tdSql.query(f"select min(c1),tbname from {dbname}.stb1 partition by tbname") query_data = tdSql.queryResult - + for row in query_data: - tbname = row[1] - tdSql.query(" select min(c1) from %s "%tbname) + tbname = f"{dbname}.{row[1]}" + tdSql.query(f"select min(c1) from %s "%tbname) tdSql.checkData(0,0,row[0]) - tdSql.query("select min(c1),tbname from stb1 partition by t1") + tdSql.query(f"select min(c1),tbname from {dbname}.stb1 partition by t1") query_data = tdSql.queryResult - + for row in query_data: - tbname = row[1] - tdSql.query(" select min(c1) from %s "%tbname) + tbname = f"{dbname}.{row[1]}" + tdSql.query(f"select min(c1) from %s "%tbname) tdSql.checkData(0,0,row[0]) - # nest query for support max - tdSql.query("select abs(c2+2)+1 from (select min(c1) c2 from stb1)") + # nest query for support min + tdSql.query(f"select abs(c2+2)+1 from (select min(c1) c2 from {dbname}.stb1)") tdSql.checkData(0,0,3.000000000) - tdSql.query("select min(c1+2)+1 as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.query(f"select min(c1+2)+1 as c2 from (select ts ,c1 ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,3.000000000) - tdSql.query("select min(a+2)+1 as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.query(f"select min(a+2)+1 as c2 from (select ts ,abs(c1) a ,c2 from {dbname}.stb1)") tdSql.checkData(0,0,3.000000000) # mixup with other functions - tdSql.query("select max(c1),count(c1),last(c2,c3),min(c1) from stb1") + tdSql.query(f"select max(c1),count(c1),last(c2,c3) from {dbname}.stb1") tdSql.checkData(0,0,28) tdSql.checkData(0,1,184) tdSql.checkData(0,2,-99999) tdSql.checkData(0,3,-999) - tdSql.checkData(0,4,0) def run(self): @@ -303,7 +274,7 @@ class TDTestCase: self.check_min_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index d2cbba19b2..15bebd5593 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -80,6 +80,12 @@ python3 ./test.py -f 2-query/distribute_agg_apercentile.py python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R python3 ./test.py -f 2-query/distribute_agg_avg.py python3 ./test.py -f 2-query/distribute_agg_avg.py -R +python3 ./test.py -f 2-query/distribute_agg_count.py +python3 ./test.py -f 2-query/distribute_agg_count.py -R +python3 ./test.py -f 2-query/distribute_agg_max.py +python3 ./test.py -f 2-query/distribute_agg_max.py -R +python3 ./test.py -f 2-query/distribute_agg_min.py +python3 ./test.py -f 2-query/distribute_agg_min.py -R @@ -146,9 +152,6 @@ python3 ./test.py -f 2-query/function_stateduration.py python3 ./test.py -f 2-query/statecount.py python3 ./test.py -f 2-query/tail.py python3 ./test.py -f 2-query/ttl_comment.py -python3 ./test.py -f 2-query/distribute_agg_count.py -python3 ./test.py -f 2-query/distribute_agg_max.py -python3 ./test.py -f 2-query/distribute_agg_min.py python3 ./test.py -f 2-query/distribute_agg_sum.py python3 ./test.py -f 2-query/distribute_agg_spread.py python3 ./test.py -f 2-query/distribute_agg_stddev.py From 6740dc1077550b7f0a253dfae2406b5d74497b7e Mon Sep 17 00:00:00 2001 From: cpwu Date: Mon, 25 Jul 2022 11:16:05 +0800 Subject: [PATCH 03/43] add case --- tests/system-test/1-insert/mutil_stage.py | 201 ++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 tests/system-test/1-insert/mutil_stage.py diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py new file mode 100644 index 0000000000..6813a67808 --- /dev/null +++ b/tests/system-test/1-insert/mutil_stage.py @@ -0,0 +1,201 @@ +import datetime + +from dataclasses import dataclass, field +from typing import List, Any, Tuple +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +PRIMARY_COL = "ts" + +INT_COL = "c_int" +BINT_COL = "c_bint" +SINT_COL = "c_sint" +TINT_COL = "c_tint" +FLOAT_COL = "c_float" +DOUBLE_COL = "c_double" +BOOL_COL = "c_bool" +TINT_UN_COL = "c_utint" +SINT_UN_COL = "c_usint" +BINT_UN_COL = "c_ubint" +INT_UN_COL = "c_uint" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" + +NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +CHAR_COL = [BINARY_COL, NCHAR_COL, ] +BOOLEAN_COL = [BOOL_COL, ] +TS_TYPE_COL = [TS_COL, ] + +INT_TAG = "t_int" + +ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL] +TAG_COL = [INT_TAG] +# insert data args： +TIME_STEP = 10000 +NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + +# init db/table +DBNAME = "db" +STBNAME = f"{DBNAME}.stb1" +CTBNAME = f"{DBNAME}.ct1" +NTBNAME = f"{DBNAME}.nt1" + +L0 = 0 +L1 = 1 +L2 = 2 + +PRIMARY_DIR = 1 +NON_PRIMARY_DIR = 0 + +DATA_PRE0 = f"data0" +DATA_PRE1 = f"data1" +DATA_PRE2 = f"data2" + +@dataclass +class DataSet: + ts_data : List[int] = field(default_factory=list) + int_data : List[int] = field(default_factory=list) + bint_data : List[int] = field(default_factory=list) + sint_data : List[int] = field(default_factory=list) + tint_data : List[int] = field(default_factory=list) + int_un_data : List[int] = field(default_factory=list) + bint_un_data: List[int] = field(default_factory=list) + sint_un_data: List[int] = field(default_factory=list) + tint_un_data: List[int] = field(default_factory=list) + float_data : List[float] = field(default_factory=list) + double_data : List[float] = field(default_factory=list) + bool_data : List[int] = field(default_factory=list) + binary_data : List[str] = field(default_factory=list) + nchar_data : List[str] = field(default_factory=list) + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + self.taos_cfg_path = tdDnodes.dnodes[0].cfgPath + self.taos_data_dir = tdDnodes.dnodes[0].dataDir + + + def cfg(self, filename, **update_dict): + self.del_old_datadir(filename) + cmd = "echo " + for k, v in update_dict.items(): + cmd += f"{k} {v}\n" + + cmd += f" >> {filename}" + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def cfg_str(self, filename, update_str): + self.del_old_datadir(filename) + cmd = f'echo "{update_str}" >> {filename}' + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def cfg_str_list(self, filename, update_list): + for update_str in update_list: + self.cfg_str(filename, update_str) + + def del_old_datadir(self, filename): + cmd = f"sed -i '/^dataDir/d' {filename}" + if os.system(cmd) != 0: + tdLog.exit(cmd) + + @property + def __err_cfg(self): + cfg_list = [] + err_case1 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" + ] + err_case2 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {PRIMARY_DIR}" + ] + err_case3 = [ + f"dataDir {self.taos_data_dir}/data33 3 {NON_PRIMARY_DIR}" + ] + err_case4 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L1} {NON_PRIMARY_DIR}" + ] + err_case5 = [] + for i in range(17): + err_case5.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i} {L0} {NON_PRIMARY_DIR}") + + err_case6 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {PRIMARY_DIR}", + ] + err_case7 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {PRIMARY_DIR}", + ] + err_case8 = [ + f"dataDir {self.taos_data_dir}/data33 3 {PRIMARY_DIR}" + ] + err_case9 = [ + f"dataDir {self.taos_data_dir}/data33 -1 {NON_PRIMARY_DIR}" + ] + + cfg_list.append(err_case1) + cfg_list.append(err_case2) + cfg_list.append(err_case3) + cfg_list.append(err_case4) + cfg_list.append(err_case5) + cfg_list.append(err_case6) + cfg_list.append(err_case7) + cfg_list.append(err_case8) + cfg_list.append(err_case9) + + return cfg_list + + @property + def __current_cfg(self): + cfg_list = [] + current_case1 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_COL}", + f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} 0", + f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} 0", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} 0" + ] + + current_case2 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 0 1", + f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 0 1", + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 0 1", + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 0 1", + + ] + + cfg_list.append(current_case1) + + return cfg_list + + def cfg_check(self): + for cfg_case in self.__err_cfg: + tdDnodes.stop(1) + self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) + tdDnodes.start(1) + + tdSql.error(f"show databases") + + + def run(self): + self.cfg_check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 76f0747bb3a15ddaef2ca56b76efaac46aee11b7 Mon Sep 17 00:00:00 2001 From: cpwu Date: Mon, 25 Jul 2022 16:08:15 +0800 Subject: [PATCH 04/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 6813a67808..4088649b2b 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -82,7 +82,6 @@ class TDTestCase: def cfg(self, filename, **update_dict): - self.del_old_datadir(filename) cmd = "echo " for k, v in update_dict.items(): cmd += f"{k} {v}\n" @@ -92,7 +91,6 @@ class TDTestCase: tdLog.exit(cmd) def cfg_str(self, filename, update_str): - self.del_old_datadir(filename) cmd = f'echo "{update_str}" >> {filename}' if os.system(cmd) != 0: tdLog.exit(cmd) @@ -183,6 +181,8 @@ class TDTestCase: def cfg_check(self): for cfg_case in self.__err_cfg: + tdLog.info("---------", self.__err_cfg.index(cfg_case)) + self.del_old_datadir(filename=self.taos_cfg_path) tdDnodes.stop(1) self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) tdDnodes.start(1) From 592041c65c101a43ea386f51bead0265c1f83134 Mon Sep 17 00:00:00 2001 From: cpwu Date: Mon, 25 Jul 2022 16:09:07 +0800 Subject: [PATCH 05/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 4088649b2b..db8c43b009 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -185,7 +185,7 @@ class TDTestCase: self.del_old_datadir(filename=self.taos_cfg_path) tdDnodes.stop(1) self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) - tdDnodes.start(1) + tdDnodes.starttaosd(1) tdSql.error(f"show databases") From f3147f201e72ab30c5486337cb669cbe894d84e1 Mon Sep 17 00:00:00 2001 From: cpwu Date: Mon, 25 Jul 2022 16:47:45 +0800 Subject: [PATCH 06/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 55 +++++++++-------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index db8c43b009..179abeff8a 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -1,11 +1,11 @@ import datetime -from dataclasses import dataclass, field from typing import List, Any, Tuple from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +from util.common import * PRIMARY_COL = "ts" @@ -54,24 +54,6 @@ DATA_PRE0 = f"data0" DATA_PRE1 = f"data1" DATA_PRE2 = f"data2" -@dataclass -class DataSet: - ts_data : List[int] = field(default_factory=list) - int_data : List[int] = field(default_factory=list) - bint_data : List[int] = field(default_factory=list) - sint_data : List[int] = field(default_factory=list) - tint_data : List[int] = field(default_factory=list) - int_un_data : List[int] = field(default_factory=list) - bint_un_data: List[int] = field(default_factory=list) - sint_un_data: List[int] = field(default_factory=list) - tint_un_data: List[int] = field(default_factory=list) - float_data : List[float] = field(default_factory=list) - double_data : List[float] = field(default_factory=list) - bool_data : List[int] = field(default_factory=list) - binary_data : List[str] = field(default_factory=list) - nchar_data : List[str] = field(default_factory=list) - - class TDTestCase: def init(self, conn, logSql): @@ -126,9 +108,9 @@ class TDTestCase: f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L1} {NON_PRIMARY_DIR}" ] - err_case5 = [] - for i in range(17): - err_case5.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i} {L0} {NON_PRIMARY_DIR}") + err_case5 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + for i in range(16): + err_case5.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") err_case6 = [ f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", @@ -161,19 +143,15 @@ class TDTestCase: def __current_cfg(self): cfg_list = [] current_case1 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_COL}", - f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} 0", - f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} 0", - f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} 0" + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" ] - current_case2 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 0 1", - f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 0 1", - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 0 1", - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 0 1", - - ] + current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + for i in range(9): + current_case2.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") cfg_list.append(current_case1) @@ -181,7 +159,7 @@ class TDTestCase: def cfg_check(self): for cfg_case in self.__err_cfg: - tdLog.info("---------", self.__err_cfg.index(cfg_case)) + tdLog.info(self.__err_cfg.index(cfg_case)) self.del_old_datadir(filename=self.taos_cfg_path) tdDnodes.stop(1) self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) @@ -189,6 +167,15 @@ class TDTestCase: tdSql.error(f"show databases") + for cfg_case in self.__current_cfg: + self.del_old_datadir(filename=self.taos_cfg_path) + tdDnodes.stop(1) + self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) + tdDnodes.starttaosd(1) + + tdSql.query(f"show databases") + + def run(self): self.cfg_check() From 3d92db9d5aa728f3dfab80613c767179927062c1 Mon Sep 17 00:00:00 2001 From: cpwu Date: Mon, 25 Jul 2022 16:48:45 +0800 Subject: [PATCH 07/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 179abeff8a..842e5a5167 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -171,7 +171,7 @@ class TDTestCase: self.del_old_datadir(filename=self.taos_cfg_path) tdDnodes.stop(1) self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) - tdDnodes.starttaosd(1) + tdDnodes.start(1) tdSql.query(f"show databases") From e6bf3ccb04aa3a507a8297a42d370deb9f727de2 Mon Sep 17 00:00:00 2001 From: cpwu Date: Mon, 25 Jul 2022 16:58:51 +0800 Subject: [PATCH 08/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 842e5a5167..d5c1c5d690 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime from typing import List, Any, Tuple from util.log import * @@ -35,7 +35,7 @@ ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE TAG_COL = [INT_TAG] # insert data args： TIME_STEP = 10000 -NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) +NOW = int(datetime.timestamp(datetime.now()) * 1000) # init db/table DBNAME = "db" From 5b06d8dff6a02461ff3daedd83a76c8fcae78bbb Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 14:31:36 +0800 Subject: [PATCH 09/43] fix case --- tests/pytest/util/sql.py | 21 +++--- tests/system-test/1-insert/mutil_stage.py | 81 +++++++++++++++++++++-- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 01955ec93a..b9177d2269 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -49,18 +49,23 @@ class TDSql: def close(self): self.cursor.close() - def prepare(self): - tdLog.info("prepare database:db") + def prepare(self, dbname="db", drop=True, **kwargs): + tdLog.info(f"prepare database:{dbname}") s = 'reset query cache' try: self.cursor.execute(s) except: tdLog.notice("'reset query cache' is not supported") - s = 'drop database if exists db' + if drop: + s = f'drop database if exists {dbname}' + self.cursor.execute(s) + s = f'create database {dbname}' + for k, v in kwargs.items(): + s += f" {k} {v}" + if "duration" not in kwargs: + s += " duration 300" self.cursor.execute(s) - s = 'create database db duration 300' - self.cursor.execute(s) - s = 'use db' + s = f'use {dbname}' self.cursor.execute(s) time.sleep(2) @@ -106,7 +111,7 @@ class TDSql: if row_tag: return self.queryResult return self.queryRows - except Exception as e: + except Exception as e: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, repr(e)) tdLog.notice("%s(%d) failed: sql:%s, %s" % args) @@ -304,7 +309,7 @@ class TDSql: tdLog.notice("Try to execute sql again, query times: %d "%i) time.sleep(1) pass - else: + else: try: tdLog.notice("Try the last execute sql ") self.affectedRows = self.cursor.execute(sql) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index d5c1c5d690..7fac180057 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -1,4 +1,5 @@ from datetime import datetime +import time from typing import List, Any, Tuple from util.log import * @@ -31,7 +32,6 @@ TS_TYPE_COL = [TS_COL, ] INT_TAG = "t_int" -ALL_COL = [PRIMARY_COL, INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BINARY_COL, NCHAR_COL, BOOL_COL, TS_COL] TAG_COL = [INT_TAG] # insert data args： TIME_STEP = 10000 @@ -39,9 +39,9 @@ NOW = int(datetime.timestamp(datetime.now()) * 1000) # init db/table DBNAME = "db" -STBNAME = f"{DBNAME}.stb1" -CTBNAME = f"{DBNAME}.ct1" -NTBNAME = f"{DBNAME}.nt1" +STBNAME = "stb1" +CTB_PRE = "ct" +NTB_PRE = "nt" L0 = 0 L1 = 1 @@ -153,32 +153,99 @@ class TDTestCase: for i in range(9): current_case2.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") + # TD-17773bug + current_case3 = [ + f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", + f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE1}0 {L1} {NON_PRIMARY_DIR}", + f"dataDir {self.taos_data_dir}/{DATA_PRE2}0 {L2} {NON_PRIMARY_DIR}", + ] cfg_list.append(current_case1) + cfg_list.append(current_case3) return cfg_list def cfg_check(self): for cfg_case in self.__err_cfg: - tdLog.info(self.__err_cfg.index(cfg_case)) self.del_old_datadir(filename=self.taos_cfg_path) tdDnodes.stop(1) + tdDnodes.deploy(1) self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) tdDnodes.starttaosd(1) - + time.sleep(2) tdSql.error(f"show databases") for cfg_case in self.__current_cfg: self.del_old_datadir(filename=self.taos_cfg_path) tdDnodes.stop(1) + tdDnodes.deploy(1) self.cfg_str_list(filename=self.taos_cfg_path, update_list=cfg_case) tdDnodes.start(1) - tdSql.query(f"show databases") + def __create_tb(self, stb=STBNAME, ctb_pre = CTB_PRE, ctb_num=20, ntb_pre=NTB_PRE, ntbnum=1, dbname=DBNAME): + tdLog.printNoPrefix("==========step: create table") + create_stb_sql = f'''create table {dbname}.{stb}( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, + {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, + {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned + ) tags ({INT_TAG} int) + ''' + for i in range(ntbnum): + + create_ntb_sql = f'''create table {dbname}.{ntb_pre}{i+1}( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, + {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, + {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(ctb_num): + tdSql.execute(f'create table {dbname}.{ctb_pre}{i+1} using {dbname}.{stb} tags ( {i+1} )') + + def __insert_data(self, rows,dbname=DBNAME): + data = DataSet().get_order_set(rows) + + tdLog.printNoPrefix("==========step: start inser data into tables now.....") + + # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + + for i in range(self.rows): + row_data = f''' + {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, + {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.tint_un_data[i]}, + {data.sint_un_data[i]}, {data.int_un_data[i]}, {data.bint_un_data[i]} + ''' + neg_row_data = f''' + {-1 * data.int_data[i]}, {-1 * data.bint_data[i]}, {-1 * data.sint_data[i]}, {-1 * data.tint_data[i]}, {-1 * data.float_data[i]}, {-1 * data.double_data[i]}, + {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.tint_un_data[i]}, + {1 * data.sint_un_data[i]}, {1 * data.int_un_data[i]}, {1 * data.bint_un_data[i]} + ''' + + tdSql.execute( + f"insert into {dbname}.{CTB_PRE}1 values ( {NOW - i * TIME_STEP}, {row_data} )") + tdSql.execute( + f"insert into {dbname}.{CTB_PRE}2 values ( {NOW - i * int(TIME_STEP * 0.6)}, {neg_row_data} )") + tdSql.execute( + f"insert into {dbname}.{CTB_PRE}4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") + tdSql.execute( + f"insert into {dbname}.{NTB_PRE}1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") + def run(self): + self.rows = 10 self.cfg_check() + tdSql.prepare(dbname=DBNAME, **{"keep": "5m, 10m, 15m", "duration":"5m"}) + self.__create_tb(dbname=DBNAME) + self.__insert_data(rows=self.rows, dbname=DBNAME) + tdSql.query(f"select count(*) from {DBNAME}.{NTB_PRE}1") def stop(self): tdSql.close() From 2cb144664e927fbb8444a4feb97fb534956f2abb Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 15:15:54 +0800 Subject: [PATCH 10/43] fix case --- tests/pytest/util/common.py | 27 +++++++++-------------- tests/system-test/1-insert/mutil_stage.py | 13 ++++------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 9b72312028..a290a92876 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -80,23 +80,18 @@ class DataSet: self.bool_data.append( bool((i + bool_start) % 2 )) self.vchar_data.append( f"{vchar_prefix}_{i * vchar_step}" ) self.nchar_data.append( f"{nchar_prefix}_{i * nchar_step}") - self.ts_data.append( int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000 - i * ts_step)) + self.ts_data.append( int(datetime.timestamp(datetime.now()) * 1000 - i * ts_step)) - def get_disorder_set(self, - rows, - int_low :int = INT_MIN, - int_up :int = INT_MAX, - bint_low :int = BIGINT_MIN, - bint_up :int = BIGINT_MAX, - sint_low :int = SMALLINT_MIN, - sint_up :int = SMALLINT_MAX, - tint_low :int = TINYINT_MIN, - tint_up :int = TINYINT_MAX, - ubint_low :int = BIGINT_UN_MIN, - ubint_up :int = BIGINT_UN_MAX, - - - ): + def get_disorder_set(self, rows, **kwargs): + for k, v in kwargs.items(): + int_low = v if k == "int_low" else INT_MIN + int_up = v if k == "int_up" else INT_MAX + bint_low = v if k == "bint_low" else BIGINT_MIN + bint_up = v if k == "bint_up" else BIGINT_MAX + sint_low = v if k == "sint_low" else SMALLINT_MIN + sint_up = v if k == "sint_up" else SMALLINT_MAX + tint_low = v if k == "tint_low" else TINYINT_MIN + tint_up = v if k == "tint_up" else TINYINT_MAX pass diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 7fac180057..6584c4081b 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -193,8 +193,9 @@ class TDTestCase: {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned ) tags ({INT_TAG} int) ''' - for i in range(ntbnum): + tdSql.execute(create_stb_sql) + for i in range(ntbnum): create_ntb_sql = f'''create table {dbname}.{ntb_pre}{i+1}( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, @@ -203,8 +204,7 @@ class TDTestCase: {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned ) ''' - tdSql.execute(create_stb_sql) - tdSql.execute(create_ntb_sql) + tdSql.execute(create_ntb_sql) for i in range(ctb_num): tdSql.execute(f'create table {dbname}.{ctb_pre}{i+1} using {dbname}.{stb} tags ( {i+1} )') @@ -213,9 +213,6 @@ class TDTestCase: data = DataSet().get_order_set(rows) tdLog.printNoPrefix("==========step: start inser data into tables now.....") - - # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) - for i in range(self.rows): row_data = f''' {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, @@ -237,12 +234,10 @@ class TDTestCase: tdSql.execute( f"insert into {dbname}.{NTB_PRE}1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") - - def run(self): self.rows = 10 self.cfg_check() - tdSql.prepare(dbname=DBNAME, **{"keep": "5m, 10m, 15m", "duration":"5m"}) + tdSql.prepare(dbname=DBNAME, **{"keep": "1d, 1500m, 26h", "duration":"1h"}) self.__create_tb(dbname=DBNAME) self.__insert_data(rows=self.rows, dbname=DBNAME) tdSql.query(f"select count(*) from {DBNAME}.{NTB_PRE}1") From c71b1440cf1501218dfa6c7ca3896ad52fd65f6c Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 15:38:54 +0800 Subject: [PATCH 11/43] fix case --- tests/pytest/util/common.py | 2 +- tests/system-test/1-insert/mutil_stage.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index a290a92876..6384195a1b 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -18,7 +18,7 @@ import time import socket import json import toml -from .boundary import DataBoundary +from util.boundary import DataBoundary import taos from util.log import * from util.sql import * diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 6584c4081b..e4342eca24 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -210,7 +210,8 @@ class TDTestCase: tdSql.execute(f'create table {dbname}.{ctb_pre}{i+1} using {dbname}.{stb} tags ( {i+1} )') def __insert_data(self, rows,dbname=DBNAME): - data = DataSet().get_order_set(rows) + data = DataSet() + data.get_order_set(rows) tdLog.printNoPrefix("==========step: start inser data into tables now.....") for i in range(self.rows): From 4d765cd64f694b0c56c0d500bb93fdce99c1aaa5 Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 17:43:53 +0800 Subject: [PATCH 12/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index e4342eca24..f18098267b 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -217,12 +217,12 @@ class TDTestCase: for i in range(self.rows): row_data = f''' {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, - {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.tint_un_data[i]}, + {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.tint_un_data[i]}, {data.sint_un_data[i]}, {data.int_un_data[i]}, {data.bint_un_data[i]} ''' neg_row_data = f''' {-1 * data.int_data[i]}, {-1 * data.bint_data[i]}, {-1 * data.sint_data[i]}, {-1 * data.tint_data[i]}, {-1 * data.float_data[i]}, {-1 * data.double_data[i]}, - {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.tint_un_data[i]}, + {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.tint_un_data[i]}, {1 * data.sint_un_data[i]}, {1 * data.int_un_data[i]}, {1 * data.bint_un_data[i]} ''' From 9101b2aeba08fde6eebc57f02d9bf774fadf9e4a Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 17:47:07 +0800 Subject: [PATCH 13/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index f18098267b..880e638798 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -217,13 +217,13 @@ class TDTestCase: for i in range(self.rows): row_data = f''' {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, - {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.tint_un_data[i]}, - {data.sint_un_data[i]}, {data.int_un_data[i]}, {data.bint_un_data[i]} + {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.utint_data[i]}, + {data.usint_data[i]}, {data.uint_data[i]}, {data.ubint_data[i]} ''' neg_row_data = f''' {-1 * data.int_data[i]}, {-1 * data.bint_data[i]}, {-1 * data.sint_data[i]}, {-1 * data.tint_data[i]}, {-1 * data.float_data[i]}, {-1 * data.double_data[i]}, - {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.tint_un_data[i]}, - {1 * data.sint_un_data[i]}, {1 * data.int_un_data[i]}, {1 * data.bint_un_data[i]} + {data.bool_data[i]}, '{data.vchar_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.utint_data[i]}, + {1 * data.usint_data[i]}, {1 * data.uint_data[i]}, {1 * data.ubint_data[i]} ''' tdSql.execute( From 4bec9b1cdc6bcf09bd35b318722addffbdc77966 Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 17:56:29 +0800 Subject: [PATCH 14/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 880e638798..f537c0042a 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -163,6 +163,9 @@ class TDTestCase: cfg_list.append(current_case1) cfg_list.append(current_case3) + # case2 must in last of least, because use this cfg as data uniformity test + cfg_list.append(current_case2) + return cfg_list def cfg_check(self): From 4cf20418cdd0d446517184077521c49ed349f562 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 26 Jul 2022 18:00:50 +0800 Subject: [PATCH 15/43] feat: support batch fetch in catalog --- include/common/tmsg.h | 22 ++ include/common/tmsgdef.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/vnode/src/inc/vnd.h | 4 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 135 +++++++- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 +- source/libs/catalog/inc/catalogInt.h | 18 + source/libs/catalog/inc/ctgRemote.h | 3 +- source/libs/catalog/src/ctgAsync.c | 21 +- source/libs/catalog/src/ctgRemote.c | 363 ++++++++++++++++++-- source/libs/catalog/src/ctgUtil.c | 34 +- source/libs/qcom/src/querymsg.c | 1 + 12 files changed, 559 insertions(+), 52 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0d0eb841bc..9d23557ee7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3045,6 +3045,28 @@ typedef struct SDeleteRes { int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); +typedef struct { + int32_t msgType; + int32_t msgLen; + void* msg; +} SBatchMsg; + +typedef struct { + int32_t reqType; + int32_t msgLen; + int32_t rspCode; + void* msg; +} SBatchRsp; + +static FORCE_INLINE void tFreeSBatchRsp(void *p) { + if (NULL == p) { + return; + } + + SBatchRsp* pRsp = (SBatchRsp*); + taosMemoryFree(pRsp->msg); +} + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 56e0935ce1..2d5a703b62 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -180,6 +180,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLE_CFG, "vnode-table-cfg", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_BATCH_META, "vnode-batch-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 0471e2b850..9ef2350b6b 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -336,6 +336,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_UPDATE_TAG_VAL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_CFG, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 984b34814d..44281ea38e 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -78,8 +78,8 @@ void vnodeBufPoolReset(SVBufPool* pPool); // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); void vnodeQueryClose(SVnode* pVnode); -int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg); -int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg); +int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct); +int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 1c3e2f0514..bf846dba13 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -21,7 +21,7 @@ int vnodeQueryOpen(SVnode *pVnode) { void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); } -int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { +int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { STableInfoReq infoReq = {0}; STableMetaRsp metaRsp = {0}; SMetaReader mer1 = {0}; @@ -99,7 +99,12 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - pRsp = rpcMallocCont(rspLen); + if (direct) { + pRsp = rpcMallocCont(rspLen); + } else { + pRsp = taosMemoryCalloc(1, rspLen); + } + if (pRsp == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -117,15 +122,17 @@ _exit: qError("get table %s meta failed cause of %s", infoReq.tbName, tstrerror(code)); } - tmsgSendRsp(&rpcMsg); - + if (direct) { + tmsgSendRsp(&rpcMsg); + } + taosMemoryFree(metaRsp.pSchemas); metaReaderClear(&mer2); metaReaderClear(&mer1); return TSDB_CODE_SUCCESS; } -int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg) { +int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { STableCfgReq cfgReq = {0}; STableCfgRsp cfgRsp = {0}; SMetaReader mer1 = {0}; @@ -209,7 +216,12 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - pRsp = rpcMallocCont(rspLen); + if (direct) { + pRsp = rpcMallocCont(rspLen); + } else { + pRsp = taosMemoryCalloc(1, rspLen); + } + if (pRsp == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -227,14 +239,121 @@ _exit: qError("get table %s cfg failed cause of %s", cfgReq.tbName, tstrerror(code)); } - tmsgSendRsp(&rpcMsg); - + if (direct) { + tmsgSendRsp(&rpcMsg); + } + tFreeSTableCfgRsp(&cfgRsp); metaReaderClear(&mer2); metaReaderClear(&mer1); return TSDB_CODE_SUCCESS; } +int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + int32_t offset = 0; + int32_t rspSize = 0; + int32_t msgNum = ntohl(pMsg->pCont); + offset += sizeof(msgNum); + SBatchMsg req = {0}; + SBatchRsp rsp = {0}; + SRpcMsg reqMsg = *pMsg; + SRpcMsg rspMsg = {0}; + void* pRsp = NULL; + + SArray* batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); + if (NULL == batchRsp) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < msgNum; ++i) { + req.msgType = ntohl((char*)pMsg->pCont + offset); + offset += req.msgType; + + req.msgLen = ntohl((char*)pMsg->pCont + offset); + offset += req.msgLen; + + req.msg = (char*)pMsg->pCont + offset; + offset += req.msgLen; + + reqMsg.msgType = req.msgType; + reqMsg.pCont = req.msg; + reqMsg.contLen = req.msgLen; + + switch (req.msgType) { + case TDMT_VND_TABLE_META: + vnodeGetTableMeta(pVnode, &reqMsg, false); + break; + case TDMT_VND_TABLE_CFG: + vnodeGetTableCfg(pVnode, &reqMsg, false); + break; + default: + qError("invalid req msgType %d", req.msgType); + reqMsg.code = TSDB_CODE_INVALID_MSG; + reqMsg.pCont = NULL; + reqMsg.contLen = 0; + break; + } + + rsp.reqType = reqMsg.msgType; + rsp.msgLen = reqMsg.contLen; + rsp.rspCode = reqMsg.code; + rsp.msg = reqMsg.pCont; + + taosArrayPush(batchRsp, &rsp); + + rspSize += sizeof(rsp) + rsp.msgLen - POINTER_BYTES; + } + + rspSize += sizeof(int32_t); + offset = 0; + + pRsp = rpcMallocCont(rspSize); + if (pRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + *(int32_t*)((char*)pRsp + offset) = htonl(msgNum); + offset += sizeof(msgNum); + for (int32_t i = 0; i < msgNum; ++i) { + SBatchRsp *p = taosArrayGet(batchRsp, i); + + *(int32_t*)((char*)pRsp + offset) = htonl(p->reqType); + offset += sizeof(p->reqType); + *(int32_t*)((char*)pRsp + offset) = htonl(p->msgLen); + offset += sizeof(p->msgLen); + *(int32_t*)((char*)pRsp + offset) = htonl(p->rspCode); + offset += sizeof(p->rspCode); + memcpy((char*)pRsp + offset, p->msg, p->msgLen); + offset += p->msgLen; + + taosMemoryFreeClear(p->msg); + } + + taosArrayDestroy(batchRsp); + batchRsp = NULL; + +_exit: + + rspMsg.info = pMsg->info; + rspMsg.pCont = pRsp; + rspMsg.contLen = rspSize; + rspMsg.code = code; + rspMsg.msgType = pMsg->msgType; + + if (code) { + qError("get batch meta failed cause of %s", tstrerror(code)); + } + + taosArrayDestroyEx(batchRsp, tFreeSBatchRsp); + + tmsgSendRsp(&rspMsg); + + return TSDB_CODE_SUCCESS; +} + int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); pLoad->syncState = syncGetMyRole(pVnode->sync); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9d06fbffdd..830ab6e37c 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -296,7 +296,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || - pMsg->msgType == TDMT_VND_TABLE_CFG) && + pMsg->msgType == TDMT_VND_TABLE_CFG || pMsg->msgType = TDMT_VND_BATCH_META) && !vnodeIsLeader(pVnode)) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; @@ -318,9 +318,11 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_SCH_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_VND_TABLE_META: - return vnodeGetTableMeta(pVnode, pMsg); + return vnodeGetTableMeta(pVnode, pMsg, true); case TDMT_VND_TABLE_CFG: - return vnodeGetTableCfg(pVnode, pMsg); + return vnodeGetTableCfg(pVnode, pMsg, true); + case TDMT_VND_BATCH_META: + return vnodeGetBatchMeta(pVnode, pMsg); case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index bf3bc1f0f4..4bd85792cd 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -31,6 +31,7 @@ extern "C" { #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 #define CTG_DEFAULT_MAX_RETRY_TIMES 3 +#define CTG_DEFAULT_BATCH_NUM 64 #define CTG_RENT_SLOT_SECOND 1.5 @@ -38,6 +39,8 @@ extern "C" { #define CTG_ERR_CODE_TABLE_NOT_EXIST TSDB_CODE_PAR_TABLE_NOT_EXIST +#define CTG_BATCH_FETCH 1 + enum { CTG_READ = 1, CTG_WRITE, @@ -200,8 +203,20 @@ typedef struct SCatalog { SCtgRentMgmt stbRent; } SCatalog; +typedef struct SCtgBatch { + int32_t batchId; + int32_t msgType; + int32_t msgSize; + SArray* pMsgs; + SRequestConnInfo *pConn; + char dbFName[TSDB_DB_FNAME_LEN]; + SArray* pTaskIds; +} SCtgBatch; + typedef struct SCtgJob { int64_t refId; + int32_t batchId; + SHashObj* pBatchs; SArray* pTasks; int32_t taskDone; SMetaData jobRes; @@ -258,6 +273,7 @@ typedef struct SCtgTask { SRWLatch lock; SArray* pParents; SCtgSubRes subRes; + SHashObj* pBatchs; } SCtgTask; typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*); @@ -626,6 +642,8 @@ int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, int32_t ctgGetTbCfgCb(SCtgTask *pTask); void ctgFreeHandle(SCatalog* pCatalog); +void ctgFreeBatch(SCtgBatch *pBatch); +void ctgFreeBatchs(SHashObj *pBatchs); int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst); int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput); int32_t ctgGenerateVgList(SCatalog *pCtg, SHashObj *vgHash, SArray** pList); diff --git a/source/libs/catalog/inc/ctgRemote.h b/source/libs/catalog/inc/ctgRemote.h index cd88863c1b..72ab43e085 100644 --- a/source/libs/catalog/inc/ctgRemote.h +++ b/source/libs/catalog/inc/ctgRemote.h @@ -23,8 +23,9 @@ extern "C" { typedef struct SCtgTaskCallbackParam { uint64_t queryId; int64_t refId; - uint64_t taskId; + SArray* taskId; int32_t reqType; + int32_t batchId; } SCtgTaskCallbackParam; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 920acbac2e..65c6a0f76b 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -473,8 +473,15 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const pJob->tbCfgNum = tbCfgNum; pJob->svrVerNum = svrVerNum; - pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask)); +#if CTG_BATCH_FETCH + pJob->pBatchs = taosHashInit(CTG_DEFAULT_BATCH_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pJob->pBatchs) { + ctgError("taosHashInit %d batch failed", CTG_DEFAULT_BATCH_NUM); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } +#endif + pJob->pTasks = taosArrayInit(taskNum, sizeof(SCtgTask)); if (NULL == pJob->pTasks) { ctgError("taosArrayInit %d tasks failed", taskNum); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); @@ -560,7 +567,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const _return: - taosMemoryFreeClear(*job); + ctgFreeJob(*job); CTG_RET(code); } @@ -872,7 +879,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * SVgroupInfo vgInfo = {0}; CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); ctx->vgId = vgInfo.vgId; CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); @@ -890,7 +897,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * return TSDB_CODE_SUCCESS; } - ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); + ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); @@ -1705,13 +1712,19 @@ int32_t ctgLaunchJob(SCtgJob *pJob) { qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); + pTask->status = CTG_TASK_LAUNCHED; + pTask->pBatchs = pJob->pBatchs; } if (taskNum <= 0) { qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); taosAsyncExec(ctgCallUserCb, pJob, NULL); +#if CTG_BATCH_FETCH + } else { + ctgLaunchBatchs(pJob->pCtg, pJob, pJob->pBatchs); +#endif } return TSDB_CODE_SUCCESS; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index cc5dde9298..04ffbca629 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -21,6 +21,64 @@ #include "ctgRemote.h" #include "tref.h" +int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SArray* pTaskId = cbParam->taskId; + int32_t taskNum = taosArrayGetSize(pTaskId); + SDataBuf taskMsg = *pMsg; + int32_t offset = 0; + int32_t msgNum = (pMsg->pData && (pMsg->len > 0)) ? htonl(pMsg->pData) : 0; + ASSERT(taskNum == msgNum || 0 == msgNum); + + qDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); + + offset += sizeof(msgNum); + SBatchRsp rsp = {0}; + SHashObj* pBatchs = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pBatchs) { + ctgError("taosHashInit %d batch failed", taskNum); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < taskNum; ++i) { + int32_t taskId = taosArrayGet(pTaskId, i); + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskId); + if (msgNum > 0) { + rsp.reqType = htonl(((char*)pMsg->pData) + offset); + offset += sizeof(rsp.reqType); + rsp.msgLen = htonl(((char*)pMsg->pData) + offset); + offset += sizeof(rsp.msgLen); + rsp.rspCode = htonl(((char*)pMsg->pData) + offset); + offset += sizeof(rsp.rspCode); + rsp.msg = ((char*)pMsg->pData) + offset; + offset += rsp.msgLen; + + taskMsg.msgType = rsp.reqType; + taskMsg.pData = rsp.msg; + taskMsg.len = rsp.msgLen; + } else { + rsp.reqType = -1; + taskMsg.msgType = -1; + taskMsg.pData = NULL; + taskMsg.len = 0; + } + + pTask->pBatchs = pBatchs; + + qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(taskMsg.msgType + 1)); + + (*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); + } + + CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); + +_return: + + ctgFreeBatchs(pBatchs); + CTG_RET(code); +} + + int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target) { int32_t code = 0; @@ -233,6 +291,11 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, break; } default: + if (TSDB_CODE_SUCCESS != rspCode) { + qError("Got error rsp, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(rspCode); + } + qError("invalid req type %s", TMSG_INFO(reqType)); return TSDB_CODE_APP_ERROR; } @@ -254,12 +317,17 @@ int32_t ctgHandleMsgCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { goto _return; } - SCtgTask *pTask = taosArrayGet(pJob->pTasks, cbParam->taskId); + if (TDMT_VND_BATCH_META == cbParam->reqType || TDMT_MND_BATCH_META == cbParam->reqType) { + CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode)); + } else { + int32_t taskId = taosArrayGet(cbParam->taskId, 0); + SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskId); - qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); + qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); + + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); + } - CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); - _return: taosMemoryFree(pMsg->pData); @@ -272,7 +340,7 @@ _return: } -int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsgSendInfo) { +int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int32_t msgType, SMsgSendInfo **pMsgSendInfo) { int32_t code = 0; SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { @@ -287,9 +355,10 @@ int32_t ctgMakeMsgSendInfo(SCtgTask* pTask, int32_t msgType, SMsgSendInfo **pMsg } param->reqType = msgType; - param->queryId = pTask->pJob->queryId; - param->refId = pTask->pJob->refId; - param->taskId = pTask->taskId; + param->queryId = pJob->queryId; + param->refId = pJob->refId; + param->taskId = pTaskId; + param->batchId = batchId; msgSendInfo->param = param; msgSendInfo->paramFreeFp = taosMemoryFree; @@ -307,12 +376,13 @@ _return: CTG_RET(code); } -int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTask* pTask, int32_t msgType, void *msg, uint32_t msgSize) { +int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob* pJob, SArray* pTaskId, + int32_t batchId, char* dbFName, int32_t vgId, int32_t msgType, void *msg, uint32_t msgSize) { int32_t code = 0; SMsgSendInfo *pMsgSendInfo = NULL; - CTG_ERR_JRET(ctgMakeMsgSendInfo(pTask, msgType, &pMsgSendInfo)); + CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, msgType, &pMsgSendInfo)); - ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, pTask); + ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId); pMsgSendInfo->requestId = pConn->requestId; pMsgSendInfo->requestObjRefId = pConn->requestObjRefId; @@ -328,19 +398,163 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTask* pTask CTG_ERR_JRET(code); } - ctgDebug("ctg req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pTask->pJob->queryId, msgType, TMSG_INFO(msgType)); + ctgDebug("ctg req msg sent, reqId:0x%" PRIx64 ", msg type:%d, %s", pJob->queryId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: if (pMsgSendInfo) { - taosMemoryFreeClear(pMsgSendInfo->param); - taosMemoryFreeClear(pMsgSendInfo); + destroySendMsgInfo(pMsgSendInfo); } CTG_RET(code); } +int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgTask* pTask, int32_t msgType, void *msg, uint32_t msgSize) { + int32_t code = 0; + SHashObj* pBatchs = pTask->pBatchs; + SCtgJob* pJob = pTask->pJob; + SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); + int32_t taskNum = taosArrayGetSize(pTask->pJob->pTasks); + SCtgBatch newBatch = {0}; + SBatchMsg req = {0}; + + if (NULL == pBatch) { + newBatch.pMsgs = taosArrayInit(taskNum, sizeof(SBatchMsg)); + newBatch.pTaskIds = taosArrayInit(taskNum, sizeof(int32_t)); + if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + newBatch.pConn = pConn; + + req.msgType = msgType; + req.msgLen = msgSize; + req.msg = msg; + if (NULL == taosArrayPush(newBatch.pMsgs, &req)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + if (NULL == taosArrayPush(newBatch.pTaskIds, &pTask->taskId)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + newBatch.msgSize = sizeof(req) + msgSize - POINTER_BYTES; + + if (vgId > 0) { + if (TDMT_VND_TABLE_CFG == msgType) { + SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; + tNameGetFullDbName(ctx->pName, newBatch.dbFName); + } else if (TDMT_VND_TABLE_META == msgType) { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + tNameGetFullDbName(ctx->pName, newBatch.dbFName); + } else { + ctgError("invalid vnode msgType %d", msgType); + CTG_ERR_JRET(TSDB_CODE_APP_ERROR); + } + } + + newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META; + newBatch.batchId = atomic_add_fetch_32(&pJob->batchId, 1); + + if (0 != taosHashPut(pBatchs, &vgId, sizeof(vgId), &newBatch, sizeof(newBatch))) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), newBatch->batchId, vgId); + + return TSDB_CODE_SUCCESS; + } + + req.msgType = msgType; + req.msgLen = msgSize; + req.msg = msg; + if (NULL == taosArrayPush(pBatch->pMsgs, &req)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + if (NULL == taosArrayPush(pBatch->pTaskIds, &pTask->taskId)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + pBatch->msgSize += sizeof(req) + msgSize - POINTER_BYTES; + + if (vgId > 0) { + if (TDMT_VND_TABLE_CFG == msgType) { + SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; + tNameGetFullDbName(ctx->pName, newBatch.dbFName); + } else if (TDMT_VND_TABLE_META == msgType) { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + tNameGetFullDbName(ctx->pName, newBatch.dbFName); + } else { + ctgError("invalid vnode msgType %d", msgType); + CTG_ERR_JRET(TSDB_CODE_APP_ERROR); + } + } + + ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, vgId); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgFreeBatch(&newBatch); + taosMemoryFree(msg); + + return code; +} + +int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, void** msg) { + *msg = taosMemoryMalloc(pBatch->msgSize); + if (NULL == (*msg)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + int32_t offset = 0; + int32_t num = taosArrayGetSize(pBatch->pMsgs); + *(int32_t*)((char*)(*msg) + offset) = htonl(num); + offset += sizeof(num); + + for (int32_t i = 0; i < num; ++i) { + SBatchMsg* pReq = taosArrayGet(pBatch->pMsgs, i); + *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgType); + offset += sizeof(pReq->msgType); + *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgLen); + offset += sizeof(pReq->msgLen); + memcpy((char*)(*msg) + offset, pReq->msg, pReq->msgLen); + offset += pReq->msgLen; + } + + ASSERT(pBatch->msgSize == offset); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob *pJob, SHashObj* pBatchs) { + int32_t code = 0; + void* msg = NULL; + void* p = taosHashIterate(pBatchs, NULL); + while (NULL != p) { + size_t len = 0; + int32_t* vgId = taosHashGetKey(p, &len); + SCtgBatch* pBatch = (SCtgBatch*)p; + + CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, &msg)); + CTG_ERR_JRET(ctgAsyncSendMsg(pCtg, pBatch->pConn, pJob, pBatch->pTaskIds, pBatch->batchId, + pBatch->dbFName, *vgId, pBatch->msgType, msg, pBatch->msgSize)); + + p = taosHashIterate(pBatchs, p); + } + + return TSDB_CODE_SUCCESS; + +_return: + + if (p) { + taosHashCancelIterate(pBatchs, p); + } + taosMemoryFree(msg); + + CTG_RET(code); +} + + int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask) { char *msg = NULL; int32_t msgLen = 0; @@ -361,7 +575,14 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -396,7 +617,14 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray if (pTask) { CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -436,8 +664,14 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db)); + + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -476,8 +710,14 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName)); + + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -517,7 +757,13 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -560,7 +806,13 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -600,7 +852,13 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const ch } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -640,7 +898,13 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -685,7 +949,13 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -744,7 +1014,21 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask, reqType, msg, msgLen)); + +#if CTG_BATCH_FETCH + CTG_RET(ctgAddVnodeBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); +#else + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(ctx->pName, dbFName); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->vgId, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -791,7 +1075,20 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask, reqType, msg, msgLen)); +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); +#else + SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(ctx->pName, dbFName); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -833,7 +1130,13 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S if (pTask) { CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { @@ -869,7 +1172,13 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **ou if (pTask) { CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); + if (NULL == pTaskId) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + taosArrayPush(pTaskId, &pTask->taskId); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); } SRpcMsg rpcMsg = { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 1f0f074a0f..8d33e5596f 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -19,6 +19,29 @@ #include "catalogInt.h" #include "systable.h" + +void ctgFreeBatch(SCtgBatch *pBatch) { + if (NULL == pBatch) { + return; + } + + taosArrayDestroy(pBatch->pMsgs); + taosArrayDestroy(pBatch->pTaskIds); +} + +void ctgFreeBatchs(SHashObj *pBatchs) { + void* p = taosHashIterate(pBatchs, NULL); + while (NULL != p) { + SCtgBatch* pBatch = (SCtgBatch*)p; + + ctgFreeBatch(pBatch); + + p = taosHashIterate(pBatchs, p); + } + + taosHashCleanup(pBatchs); +} + char *ctgTaskTypeStr(CTG_TASK_TYPE type) { switch (type) { case CTG_TASK_GET_QNODE: @@ -612,6 +635,7 @@ void ctgFreeJob(void* job) { uint64_t qid = pJob->queryId; ctgFreeTasks(pJob->pTasks); + ctgFreeBatchs(pJob->pBatchs); ctgFreeSMetaData(&pJob->jobRes); @@ -867,14 +891,10 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { } -int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask) { - if (msgType == TDMT_VND_TABLE_META) { - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(ctx->pName, dbFName); - +int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId) { + if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META) { pMsgSendInfo->target.type = TARGET_TYPE_VNODE; - pMsgSendInfo->target.vgId = ctx->vgId; + pMsgSendInfo->target.vgId = vgId; pMsgSendInfo->target.dbFName = strdup(dbFName); } else { pMsgSendInfo->target.type = TARGET_TYPE_MNODE; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index ed8786170d..78ecde6003 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -636,6 +636,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_TABLE_INDEX)] = queryProcessGetTbIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_CFG)] = queryProcessGetTbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_CFG)] = queryProcessGetTbCfgRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_BATCH_META)] = queryProcessGetBatchMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_SERVER_VERSION)] = queryProcessGetSerVerRsp; } From 424d4094007b0675373c2a928b6c708d3600d889 Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 18:02:22 +0800 Subject: [PATCH 16/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index f537c0042a..711f3321ff 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -241,10 +241,11 @@ class TDTestCase: def run(self): self.rows = 10 self.cfg_check() - tdSql.prepare(dbname=DBNAME, **{"keep": "1d, 1500m, 26h", "duration":"1h"}) + tdSql.prepare(dbname=DBNAME, **{"keep": "1d, 1500m, 26h", "duration":"1h", "vgroups": 10}) self.__create_tb(dbname=DBNAME) self.__insert_data(rows=self.rows, dbname=DBNAME) tdSql.query(f"select count(*) from {DBNAME}.{NTB_PRE}1") + tdSql.execute(f"flush database {DBNAME}") def stop(self): tdSql.close() From b9a23993be48897e20d5a889cbfd4ae869062b51 Mon Sep 17 00:00:00 2001 From: cpwu Date: Tue, 26 Jul 2022 18:23:02 +0800 Subject: [PATCH 17/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 711f3321ff..97a4afdc60 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -212,7 +212,7 @@ class TDTestCase: for i in range(ctb_num): tdSql.execute(f'create table {dbname}.{ctb_pre}{i+1} using {dbname}.{stb} tags ( {i+1} )') - def __insert_data(self, rows,dbname=DBNAME): + def __insert_data(self, rows, dbname=DBNAME, ctb_num=20): data = DataSet() data.get_order_set(rows) @@ -229,12 +229,14 @@ class TDTestCase: {1 * data.usint_data[i]}, {1 * data.uint_data[i]}, {1 * data.ubint_data[i]} ''' - tdSql.execute( - f"insert into {dbname}.{CTB_PRE}1 values ( {NOW - i * TIME_STEP}, {row_data} )") - tdSql.execute( - f"insert into {dbname}.{CTB_PRE}2 values ( {NOW - i * int(TIME_STEP * 0.6)}, {neg_row_data} )") - tdSql.execute( - f"insert into {dbname}.{CTB_PRE}4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") + for i in range(ctb_num): + tdSql.execute( + f"insert into {dbname}.{CTB_PRE}{i + 1} values ( {NOW - i * TIME_STEP}, {row_data} )") + + # tdSql.execute( + # f"insert into {dbname}.{CTB_PRE}2 values ( {NOW - i * int(TIME_STEP * 0.6)}, {neg_row_data} )") + # tdSql.execute( + # f"insert into {dbname}.{CTB_PRE}4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") tdSql.execute( f"insert into {dbname}.{NTB_PRE}1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") From 602d2197fb599b85c0f9d26c5ef525cbfb8be26c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 27 Jul 2022 08:27:24 +0800 Subject: [PATCH 18/43] feat: support vnode meta batch fetching --- include/common/tmsg.h | 8 +++- include/common/tmsgdef.h | 1 + source/dnode/vnode/src/inc/vnd.h | 1 + source/dnode/vnode/src/vnd/vnodeQuery.c | 17 ++++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/catalog/inc/catalogInt.h | 5 +- source/libs/catalog/src/ctgAsync.c | 2 +- source/libs/catalog/src/ctgRemote.c | 62 +++++++++++++++++-------- source/libs/qcom/src/querymsg.c | 1 - 9 files changed, 67 insertions(+), 32 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9d23557ee7..0571f8382a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3051,6 +3051,12 @@ typedef struct { void* msg; } SBatchMsg; +typedef struct { + SMsgHead header; + int32_t msgNum; + SBatchMsg msg[]; +} SBatchReq; + typedef struct { int32_t reqType; int32_t msgLen; @@ -3063,7 +3069,7 @@ static FORCE_INLINE void tFreeSBatchRsp(void *p) { return; } - SBatchRsp* pRsp = (SBatchRsp*); + SBatchRsp* pRsp = (SBatchRsp*)p; taosMemoryFree(pRsp->msg); } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 2d5a703b62..20dc04631e 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -136,6 +136,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "get-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_INDEX, "get-table-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_BATCH_META, "batch-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_CFG, "table-cfg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "alter-topic", NULL, NULL) diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 44281ea38e..dd1facb462 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -80,6 +80,7 @@ int32_t vnodeQueryOpen(SVnode* pVnode); void vnodeQueryClose(SVnode* pVnode); int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct); int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct); +int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index bf846dba13..6b36a66776 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -124,6 +124,8 @@ _exit: if (direct) { tmsgSendRsp(&rpcMsg); + } else { + *pMsg = rpcMsg; } taosMemoryFree(metaRsp.pSchemas); @@ -241,6 +243,8 @@ _exit: if (direct) { tmsgSendRsp(&rpcMsg); + } else { + *pMsg = rpcMsg; } tFreeSTableCfgRsp(&cfgRsp); @@ -253,8 +257,9 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { int32_t code = 0; int32_t offset = 0; int32_t rspSize = 0; - int32_t msgNum = ntohl(pMsg->pCont); - offset += sizeof(msgNum); + SBatchReq *batchReq = (SBatchReq*)pMsg->pCont; + int32_t msgNum = ntohl(batchReq->msgNum); + offset += sizeof(SBatchReq); SBatchMsg req = {0}; SBatchRsp rsp = {0}; SRpcMsg reqMsg = *pMsg; @@ -268,11 +273,11 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { - req.msgType = ntohl((char*)pMsg->pCont + offset); - offset += req.msgType; + req.msgType = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); + offset += sizeof(req.msgType); - req.msgLen = ntohl((char*)pMsg->pCont + offset); - offset += req.msgLen; + req.msgLen = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); + offset += sizeof(req.msgLen); req.msg = (char*)pMsg->pCont + offset; offset += req.msgLen; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 830ab6e37c..f560d6a8c2 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -296,7 +296,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("message in fetch queue is processing"); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || - pMsg->msgType == TDMT_VND_TABLE_CFG || pMsg->msgType = TDMT_VND_BATCH_META) && + pMsg->msgType == TDMT_VND_TABLE_CFG || pMsg->msgType == TDMT_VND_BATCH_META) && !vnodeIsLeader(pVnode)) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 4bd85792cd..0821a91d6b 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -208,7 +208,7 @@ typedef struct SCtgBatch { int32_t msgType; int32_t msgSize; SArray* pMsgs; - SRequestConnInfo *pConn; + SRequestConnInfo conn; char dbFName[TSDB_DB_FNAME_LEN]; SArray* pTaskIds; } SCtgBatch; @@ -634,6 +634,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask); int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask); int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask); +int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob *pJob, SHashObj* pBatchs); int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param); int32_t ctgLaunchJob(SCtgJob *pJob); @@ -660,7 +661,7 @@ int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); char * ctgTaskTypeStr(CTG_TASK_TYPE type); -int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); +int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); void ctgFreeSTableIndex(void *info); void ctgClearSubTaskRes(SCtgSubRes *pRes); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 65c6a0f76b..9a666eb4ff 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1709,12 +1709,12 @@ int32_t ctgLaunchJob(SCtgJob *pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask *pTask = taosArrayGet(pJob->pTasks, i); + pTask->pBatchs = pJob->pBatchs; qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask->status = CTG_TASK_LAUNCHED; - pTask->pBatchs = pJob->pBatchs; } if (taskNum <= 0) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 04ffbca629..8b1d125a58 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -24,13 +24,14 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SArray* pTaskId = cbParam->taskId; + SCatalog* pCtg = pJob->pCtg; int32_t taskNum = taosArrayGetSize(pTaskId); SDataBuf taskMsg = *pMsg; int32_t offset = 0; - int32_t msgNum = (pMsg->pData && (pMsg->len > 0)) ? htonl(pMsg->pData) : 0; + int32_t msgNum = (pMsg->pData && (pMsg->len > 0)) ? ntohl(*(int32_t*)pMsg->pData) : 0; ASSERT(taskNum == msgNum || 0 == msgNum); - qDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); + ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); offset += sizeof(msgNum); SBatchRsp rsp = {0}; @@ -41,14 +42,14 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } for (int32_t i = 0; i < taskNum; ++i) { - int32_t taskId = taosArrayGet(pTaskId, i); - SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskId); + int32_t* taskId = taosArrayGet(pTaskId, i); + SCtgTask *pTask = taosArrayGet(pJob->pTasks, *taskId); if (msgNum > 0) { - rsp.reqType = htonl(((char*)pMsg->pData) + offset); + rsp.reqType = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.reqType); - rsp.msgLen = htonl(((char*)pMsg->pData) + offset); + rsp.msgLen = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.msgLen); - rsp.rspCode = htonl(((char*)pMsg->pData) + offset); + rsp.rspCode = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.rspCode); rsp.msg = ((char*)pMsg->pData) + offset; offset += rsp.msgLen; @@ -65,7 +66,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu pTask->pBatchs = pBatchs; - qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(taskMsg.msgType + 1)); + ctgDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(taskMsg.msgType + 1)); (*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); } @@ -317,15 +318,30 @@ int32_t ctgHandleMsgCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { goto _return; } + SCatalog* pCtg = pJob->pCtg; + if (TDMT_VND_BATCH_META == cbParam->reqType || TDMT_MND_BATCH_META == cbParam->reqType) { CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode)); } else { - int32_t taskId = taosArrayGet(cbParam->taskId, 0); - SCtgTask *pTask = taosArrayGet(pJob->pTasks, taskId); + int32_t *taskId = taosArrayGet(cbParam->taskId, 0); + SCtgTask *pTask = taosArrayGet(pJob->pTasks, *taskId); qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); - + +#if CTG_BATCH_FETCH + SHashObj* pBatchs = taosHashInit(CTG_DEFAULT_BATCH_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + if (NULL == pBatchs) { + ctgError("taosHashInit %d batch failed", CTG_DEFAULT_BATCH_NUM); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + pTask->pBatchs = pBatchs; +#endif + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); + +#if CTG_BATCH_FETCH + CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); +#endif } _return: @@ -393,6 +409,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob* pJob, int64_t transporterId = 0; code = asyncSendMsgToServer(pConn->pTrans, &pConn->mgmtEps, &transporterId, pMsgSendInfo); + pMsgSendInfo = NULL; if (code) { ctgError("asyncSendMsgToSever failed, error: %s", tstrerror(code)); CTG_ERR_JRET(code); @@ -426,7 +443,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - newBatch.pConn = pConn; + newBatch.conn = *pConn; req.msgType = msgType; req.msgLen = msgSize; @@ -437,7 +454,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT if (NULL == taosArrayPush(newBatch.pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - newBatch.msgSize = sizeof(req) + msgSize - POINTER_BYTES; + newBatch.msgSize = sizeof(SBatchReq) + sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { if (TDMT_VND_TABLE_CFG == msgType) { @@ -459,7 +476,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), newBatch->batchId, vgId); + ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), newBatch.batchId, vgId); return TSDB_CODE_SUCCESS; } @@ -500,7 +517,7 @@ _return: return code; } -int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, void** msg) { +int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { *msg = taosMemoryMalloc(pBatch->msgSize); if (NULL == (*msg)) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -508,8 +525,11 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, void** msg) { int32_t offset = 0; int32_t num = taosArrayGetSize(pBatch->pMsgs); - *(int32_t*)((char*)(*msg) + offset) = htonl(num); - offset += sizeof(num); + SBatchReq *pBatchReq = (SBatchReq*)(*msg); + + pBatchReq->header.vgId = htonl(vgId); + pBatchReq->msgNum = htonl(num); + offset += sizeof(SBatchReq); for (int32_t i = 0; i < num; ++i) { SBatchMsg* pReq = taosArrayGet(pBatch->pMsgs, i); @@ -534,9 +554,11 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob *pJob, SHashObj* pBatchs) { size_t len = 0; int32_t* vgId = taosHashGetKey(p, &len); SCtgBatch* pBatch = (SCtgBatch*)p; + + ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); - CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, &msg)); - CTG_ERR_JRET(ctgAsyncSendMsg(pCtg, pBatch->pConn, pJob, pBatch->pTaskIds, pBatch->batchId, + CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg)); + CTG_ERR_JRET(ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->dbFName, *vgId, pBatch->msgType, msg, pBatch->msgSize)); p = taosHashIterate(pBatchs, p); @@ -1016,7 +1038,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa .mgmtEps = vgroupInfo->epSet}; #if CTG_BATCH_FETCH - CTG_RET(ctgAddVnodeBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); #else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 78ecde6003..ed8786170d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -636,7 +636,6 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_TABLE_INDEX)] = queryProcessGetTbIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_CFG)] = queryProcessGetTbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_CFG)] = queryProcessGetTbCfgRsp; - queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_BATCH_META)] = queryProcessGetBatchMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_SERVER_VERSION)] = queryProcessGetSerVerRsp; } From 1c45770c6240bbc6f04a5f76cfbbc2d7c5f884ab Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 27 Jul 2022 10:10:05 +0800 Subject: [PATCH 19/43] fix case of tSma --- tests/system-test/1-insert/time_range_wise.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py index f945bafe3b..e65dded601 100644 --- a/tests/system-test/1-insert/time_range_wise.py +++ b/tests/system-test/1-insert/time_range_wise.py @@ -1,4 +1,5 @@ import datetime +import time from dataclasses import dataclass from typing import List, Any, Tuple @@ -328,11 +329,15 @@ class TDTestCase: tdSql.query("select database()") dbname = tdSql.getData(0,0) tdSql.query("show databases") + for index , value in enumerate(tdSql.cursor.description): + if value[0] == "retention": + r_index = index + break for row in tdSql.queryResult: if row[0] == dbname: - if row[-1] is None: + if row[r_index] is None: continue - if ":" in row[-1]: + if ":" in row[r_index]: sma.rollup_db = True if sma.rollup_db : return False @@ -393,8 +398,6 @@ class TDTestCase: else: tdSql.error(self.__create_sma_index(sma)) - - def __drop_sma_index(self, sma:SMAschema): sql = f"{sma.drop} {sma.drop_flag} {sma.index_name}" return sql @@ -416,8 +419,7 @@ class TDTestCase: self.sma_created_index = list(filter(lambda x: x != sma.index_name, self.sma_created_index)) tdSql.query("show streams") tdSql.checkRows(self.sma_count) - - + time.sleep(1) else: tdSql.error(self.__drop_sma_index(sma)) From 3c8ac2192653cfe19b73e6fb6b91901e880543f7 Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 27 Jul 2022 10:10:15 +0800 Subject: [PATCH 20/43] fix case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 7c42f51ee0..7ecc81415d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -27,7 +27,7 @@ python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py python3 ./test.py -f 1-insert/insertWithMoreVgroup.py python3 ./test.py -f 1-insert/table_comment.py -#python3 ./test.py -f 1-insert/time_range_wise.py +python3 ./test.py -f 1-insert/time_range_wise.py python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py From b14dee976fa483ec6e48086c1b259301b2eeff31 Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 27 Jul 2022 10:43:50 +0800 Subject: [PATCH 21/43] fix case --- tests/system-test/1-insert/mutil_stage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 97a4afdc60..fcad114edd 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -229,9 +229,9 @@ class TDTestCase: {1 * data.usint_data[i]}, {1 * data.uint_data[i]}, {1 * data.ubint_data[i]} ''' - for i in range(ctb_num): + for j in range(ctb_num): tdSql.execute( - f"insert into {dbname}.{CTB_PRE}{i + 1} values ( {NOW - i * TIME_STEP}, {row_data} )") + f"insert into {dbname}.{CTB_PRE}{j + 1} values ( {NOW - i * TIME_STEP}, {row_data} )") # tdSql.execute( # f"insert into {dbname}.{CTB_PRE}2 values ( {NOW - i * int(TIME_STEP * 0.6)}, {neg_row_data} )") From 648a7a12b005c0a5e5c71c4b19e6da28a481d3c3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 27 Jul 2022 14:02:05 +0800 Subject: [PATCH 22/43] feat: support mnd meta batch fetching --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 1 + source/dnode/mnode/impl/src/mndQuery.c | 99 +++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeQuery.c | 4 +- source/libs/catalog/inc/catalogInt.h | 11 +++ source/libs/catalog/inc/ctgRemote.h | 7 -- source/libs/catalog/src/ctgAsync.c | 5 +- source/libs/catalog/src/ctgRemote.c | 75 ++++++++++++---- source/libs/catalog/src/ctgUtil.c | 10 +++ 8 files changed, 187 insertions(+), 25 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 22a53f07f6..647af20fcf 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -184,6 +184,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_BATCH_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 5a527b994e..e6e9cae100 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -63,6 +63,104 @@ int32_t mndProcessQueryMsg(SRpcMsg *pMsg) { return code; } +int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { + int32_t code = 0; + int32_t offset = 0; + int32_t rspSize = 0; + SBatchReq *batchReq = (SBatchReq*)pMsg->pCont; + int32_t msgNum = ntohl(batchReq->msgNum); + offset += sizeof(SBatchReq); + SBatchMsg req = {0}; + SBatchRsp rsp = {0}; + SRpcMsg reqMsg = *pMsg; + SRpcMsg rspMsg = {0}; + void* pRsp = NULL; + SMnode *pMnode = pMsg->info.node; + + SArray* batchRsp = taosArrayInit(msgNum, sizeof(SBatchRsp)); + if (NULL == batchRsp) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < msgNum; ++i) { + req.msgType = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); + offset += sizeof(req.msgType); + + req.msgLen = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); + offset += sizeof(req.msgLen); + + req.msg = (char*)pMsg->pCont + offset; + offset += req.msgLen; + + reqMsg.msgType = req.msgType; + reqMsg.pCont = req.msg; + reqMsg.contLen = req.msgLen; + reqMsg.info.rsp = NULL; + reqMsg.info.rspLen = 0; + + MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(req.msgType)]; + if (fp == NULL) { + mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + return -1; + } + + code = (*fp)(&reqMsg); + + rsp.reqType = reqMsg.msgType; + rsp.msgLen = reqMsg.info.rspLen; + rsp.rspCode = code; + rsp.msg = reqMsg.info.rsp; + + taosArrayPush(batchRsp, &rsp); + + rspSize += sizeof(rsp) + rsp.msgLen - POINTER_BYTES; + } + + rspSize += sizeof(int32_t); + offset = 0; + + pRsp = rpcMallocCont(rspSize); + if (pRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + *(int32_t*)((char*)pRsp + offset) = htonl(msgNum); + offset += sizeof(msgNum); + for (int32_t i = 0; i < msgNum; ++i) { + SBatchRsp *p = taosArrayGet(batchRsp, i); + + *(int32_t*)((char*)pRsp + offset) = htonl(p->reqType); + offset += sizeof(p->reqType); + *(int32_t*)((char*)pRsp + offset) = htonl(p->msgLen); + offset += sizeof(p->msgLen); + *(int32_t*)((char*)pRsp + offset) = htonl(p->rspCode); + offset += sizeof(p->rspCode); + memcpy((char*)pRsp + offset, p->msg, p->msgLen); + offset += p->msgLen; + + rpcFreeCont(p->msg); + } + + taosArrayDestroy(batchRsp); + batchRsp = NULL; + +_exit: + + pMsg->info.rsp = pRsp; + pMsg->info.rspLen = rspSize; + + if (code) { + mError("mnd get batch meta failed cause of %s", tstrerror(code)); + } + + taosArrayDestroyEx(batchRsp, tFreeSBatchRsp); + + return code; +} + int32_t mndInitQuery(SMnode *pMnode) { if (qWorkerInit(NODE_TYPE_MNODE, MNODE_HANDLE, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) { mError("failed to init qworker in mnode since %s", terrstr()); @@ -76,6 +174,7 @@ int32_t mndInitQuery(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_SCH_MERGE_FETCH, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_DROP_TASK, mndProcessQueryMsg); mndSetMsgHandle(pMnode, TDMT_SCH_QUERY_HEARTBEAT, mndProcessQueryMsg); + mndSetMsgHandle(pMnode, TDMT_MND_BATCH_META, mndProcessBatchMetaMsg); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6b36a66776..71b9d70518 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -349,14 +349,14 @@ _exit: rspMsg.msgType = pMsg->msgType; if (code) { - qError("get batch meta failed cause of %s", tstrerror(code)); + qError("vnd get batch meta failed cause of %s", tstrerror(code)); } taosArrayDestroyEx(batchRsp, tFreeSBatchRsp); tmsgSendRsp(&rspMsg); - return TSDB_CODE_SUCCESS; + return code; } int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 0821a91d6b..1aaa1ecfd7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -251,6 +251,16 @@ typedef struct SCtgMsgCtx { char* target; } SCtgMsgCtx; + +typedef struct SCtgTaskCallbackParam { + uint64_t queryId; + int64_t refId; + SArray* taskId; + int32_t reqType; + int32_t batchId; +} SCtgTaskCallbackParam; + + typedef struct SCtgTask SCtgTask; typedef int32_t (*ctgSubTaskCbFp)(SCtgTask*); @@ -643,6 +653,7 @@ int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, int32_t ctgGetTbCfgCb(SCtgTask *pTask); void ctgFreeHandle(SCatalog* pCatalog); +void ctgFreeMsgSendParam(void* param); void ctgFreeBatch(SCtgBatch *pBatch); void ctgFreeBatchs(SHashObj *pBatchs); int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst); diff --git a/source/libs/catalog/inc/ctgRemote.h b/source/libs/catalog/inc/ctgRemote.h index 72ab43e085..fe0762a88a 100644 --- a/source/libs/catalog/inc/ctgRemote.h +++ b/source/libs/catalog/inc/ctgRemote.h @@ -20,13 +20,6 @@ extern "C" { #endif -typedef struct SCtgTaskCallbackParam { - uint64_t queryId; - int64_t refId; - SArray* taskId; - int32_t reqType; - int32_t batchId; -} SCtgTaskCallbackParam; #ifdef __cplusplus diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 9a666eb4ff..f4cee13ec0 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -783,7 +783,8 @@ int32_t ctgCallSubCb(SCtgTask *pTask) { pParent->subRes.code = code; } } - + + pParent->pBatchs = pTask->pBatchs; CTG_ERR_JRET(pParent->subRes.fp(pParent)); } @@ -1660,6 +1661,7 @@ int32_t ctgSetSubTaskCb(SCtgTask *pSub, SCtgTask *pTask) { if (CTG_TASK_DONE == pSub->status) { pTask->subRes.code = pSub->code; CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].cloneFp)(pSub, &pTask->subRes.res)); + pTask->pBatchs = pSub->pBatchs; CTG_ERR_JRET(pTask->subRes.fp(pTask)); } else { if (NULL == pSub->pParents) { @@ -1697,6 +1699,7 @@ int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask)); if (newTask) { + pSub->pBatchs = pTask->pBatchs; CTG_ERR_RET((*gCtgAsyncFps[pSub->type].launchFp)(pSub)); pSub->status = CTG_TASK_LAUNCHED; } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 8b1d125a58..3ec30ac39f 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -18,7 +18,6 @@ #include "tname.h" #include "catalogInt.h" #include "systable.h" -#include "ctgRemote.h" #include "tref.h" int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf *pMsg, int32_t rspCode) { @@ -361,7 +360,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int3 SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } SCtgTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); @@ -377,7 +376,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int3 param->batchId = batchId; msgSendInfo->param = param; - msgSendInfo->paramFreeFp = taosMemoryFree; + msgSendInfo->paramFreeFp = ctgFreeMsgSendParam; msgSendInfo->fp = ctgHandleMsgCallback; *pMsgSendInfo = msgSendInfo; @@ -386,8 +385,8 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int3 _return: - taosMemoryFree(param); - taosMemoryFree(msgSendInfo); + taosArrayDestroy(pTaskId); + destroySendMsgInfo(msgSendInfo); CTG_RET(code); } @@ -543,6 +542,8 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { ASSERT(pBatch->msgSize == offset); + qDebug("batch req %d to vg %d msg built with %d meta reqs", pBatch->batchId, vgId, num); + return TSDB_CODE_SUCCESS; } @@ -558,9 +559,11 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob *pJob, SHashObj* pBatchs) { ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg)); - CTG_ERR_JRET(ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, - pBatch->dbFName, *vgId, pBatch->msgType, msg, pBatch->msgSize)); - + code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, + pBatch->dbFName, *vgId, pBatch->msgType, msg, pBatch->msgSize); + pBatch->pTaskIds = NULL; + CTG_ERR_JRET(code); + p = taosHashIterate(pBatchs, p); } @@ -598,6 +601,9 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL)); +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -605,6 +611,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -640,6 +647,9 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray if (pTask) { CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -647,6 +657,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -687,6 +698,9 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db)); +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -694,6 +708,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -733,6 +748,9 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName)); +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -740,6 +758,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -778,7 +797,10 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName)); - + +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -786,6 +808,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -827,7 +850,10 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); - + +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -835,6 +861,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -873,7 +900,10 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const ch CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName)); - + +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -881,6 +911,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const ch taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -919,7 +950,10 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user)); - + +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -927,6 +961,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -970,7 +1005,9 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); - +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -978,6 +1015,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -1151,7 +1189,9 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S if (pTask) { CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); - +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -1159,6 +1199,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { @@ -1193,7 +1234,10 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **ou if (pTask) { CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); - + +#if CTG_BATCH_FETCH + CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); +#else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); @@ -1201,6 +1245,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **ou taosArrayPush(pTaskId, &pTask->taskId); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); +#endif } SRpcMsg rpcMsg = { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 8d33e5596f..e61becbe17 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -19,6 +19,16 @@ #include "catalogInt.h" #include "systable.h" +void ctgFreeMsgSendParam(void* param) { + if (NULL == param) { + return; + } + + SCtgTaskCallbackParam* pParam = (SCtgTaskCallbackParam*)param; + taosArrayDestroy(pParam->taskId); + + taosMemoryFree(param); +} void ctgFreeBatch(SCtgBatch *pBatch) { if (NULL == pBatch) { From f15c6b0ef08a8607180c2c7e4722d11992ffee6c Mon Sep 17 00:00:00 2001 From: cpwu Date: Wed, 27 Jul 2022 14:38:34 +0800 Subject: [PATCH 23/43] fix case --- tests/system-test/2-query/diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/diff.py b/tests/system-test/2-query/diff.py index 9eac56b9e3..76d4891a1c 100644 --- a/tests/system-test/2-query/diff.py +++ b/tests/system-test/2-query/diff.py @@ -96,7 +96,7 @@ class TDTestCase: tdSql.error(f"select diff(col12) from {dbname}.stb_1") tdSql.error(f"select diff(col13) from {dbname}.stb_1") tdSql.error(f"select diff(col14) from {dbname}.stb_1") - tdSql.error(f"select ts,diff(col1),ts from {dbname}.stb_1") + tdSql.query(f"select ts,diff(col1),ts from {dbname}.stb_1") tdSql.query(f"select diff(col1) from {dbname}.stb_1") tdSql.checkRows(10) From e1c413000790ccd2cc065c129d7e478fbff2d281 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Jul 2022 15:23:33 +0800 Subject: [PATCH 24/43] fix:add test case --- tests/system-test/7-tmq/tmq_taosx.py | 91 +++++ tests/system-test/fulltest.sh | 1 + tests/test/c/CMakeLists.txt | 8 + tests/test/c/tmq_taosx_ci.c | 520 +++++++++++++++++++++++++++ 4 files changed, 620 insertions(+) create mode 100644 tests/system-test/7-tmq/tmq_taosx.py create mode 100644 tests/test/c/tmq_taosx_ci.c diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py new file mode 100644 index 0000000000..a136d0a1a2 --- /dev/null +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -0,0 +1,91 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -c %s'%(buildPath, cfgPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + srcFile = '%s/../log/tmq_taosx_tmp.source'%(cfgPath) + dstFile = '%s/../log/tmq_taosx_tmp.result'%(cfgPath) + tdLog.info("compare file: %s, %s"%(srcFile, dstFile)) + + consumeFile = open(srcFile, mode='r') + queryFile = open(dstFile, mode='r') + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("compare error: %s != %s"%src, dst) + else: + break + + tdSql.execute('use db_taosx') + tdSql.query("select * from ct3") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 51) + tdSql.checkData(0, 4, 940) + tdSql.checkData(1, 1, 23) + tdSql.checkData(1, 4, None) + + tdSql.query("select * from ct1") + tdSql.checkRows(4) + + tdSql.query("select * from ct2") + tdSql.checkRows(0) + + tdSql.query("select * from ct0") + tdSql.checkRows(2) + tdSql.checkData(0, 3, "a") + tdSql.checkData(1, 4, None) + + tdSql.query("select * from n1") + tdSql.checkRows(2) + tdSql.checkData(0, 1, "eeee") + tdSql.checkData(1, 2, 940) + + tdSql.query("select * from jt") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 11) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 1, 1) + tdSql.checkData(1, 2, '{"k1":1,"k2":"hello"}') + + return + + def run(self): + tdSql.prepare() + self.checkFileContent() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 57cadb7de7..c86fb59dd7 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -227,6 +227,7 @@ python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py +python3 ./test.py -f 7-tmq/tmq_taosx.py # python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py #------------querPolicy 2----------- diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 5db97a0f0f..605eef9be3 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(tmq_demo tmqDemo.c) add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) +add_executable(tmq_taosx_ci tmq_taosx_ci.c) target_link_libraries( create_table PUBLIC taos_static @@ -22,6 +23,13 @@ target_link_libraries( PUBLIC common PUBLIC os ) +target_link_libraries( + tmq_taosx_ci + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) add_executable(sdbDump sdbDump.c) target_link_libraries( diff --git a/tests/test/c/tmq_taosx_ci.c b/tests/test/c/tmq_taosx_ci.c new file mode 100644 index 0000000000..ece7ad4819 --- /dev/null +++ b/tests/test/c/tmq_taosx_ci.c @@ -0,0 +1,520 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" + +static int running = 1; +TdFilePtr g_fp = NULL; +char dir[64]={0}; + +static TAOS* use_db(){ + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +static void msg_process(TAOS_RES* msg) { + /*memset(buf, 0, 1024);*/ + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS *pConn = use_db(); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { + char* result = tmq_get_json_meta(msg); + if (result) { + printf("meta result: %s\n", result); + } + taosFprintfFile(g_fp, result); + taosFprintfFile(g_fp, "\n"); + tmq_free_json_meta(result); + } + + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + +// else{ +// while(1){ +// int numOfRows = 0; +// void *pData = NULL; +// taos_fetch_raw_block(msg, &numOfRows, &pData); +// if(numOfRows == 0) break; +// printf("write data: tbname:%s, numOfRows:%d\n", tmq_get_table_name(msg), numOfRows); +// int ret = taos_write_raw_block(pConn, numOfRows, pData, tmq_get_table_name(msg)); +// printf("write raw data: %s\n", tmq_err2str(ret)); +// } +// } + + taos_close(pConn); +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_taosx"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_taosx vgroups 1"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists abc1"); + if (taos_errno(pRes) != 0) { + printf("error in drop db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct0 using st1 tags(1000, \"ttt\", true)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct0 values(1626006833600, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct1 using st1(t1) tags(2000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct2 using st1(t1) tags(NULL)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 values(1626006833600, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists ct3 using st1(t1) tags(3000)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, 'ddd') ct0 values(1626006833602, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 modify column c3 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (1626006833609, 51, 62, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct3 select * from ct1"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table st1 add tag t2 binary(64)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table ct3 set tag t1=5000"); + if (taos_errno(pRes) != 0) { + printf("failed to slter child table ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "delete from abc1 .ct3 where ts < 1626006833606"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists n1(ts timestamp, c1 int, c2 nchar(4))"); + if (taos_errno(pRes) != 0) { + printf("failed to create normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 add column c3 bigint"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 modify column c2 nchar(8)"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 rename column c3 cc3"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 comment 'hello'"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "alter table n1 drop column c1"); + if (taos_errno(pRes) != 0) { + printf("failed to alter normal table n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into n1 values(now, 'eeee', 8989898899999) (now+9s, 'c333', 940)"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into n1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt(ts timestamp, i int) tags(t json)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt1 using jt tags('{\"k1\":1, \"k2\":\"hello\"}')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table jt2 using jt tags('')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt1 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into jt2 values(now, 11)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table jt2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { +#if 0 + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); +#endif + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.heartbeat.background", "true"); + + /*tmq_conf_set(conf, "experimental.snapshot.enable", "true");*/ + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "topic_ctb_column"); + /*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/ + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); + if (tmqmessage) { + cnt++; + msg_process(tmqmessage); + /*if (cnt >= 2) break;*/ + /*printf("get data\n");*/ + taos_free_result(tmqmessage); + /*} else {*/ + /*break;*/ + /*tmq_commit_sync(tmq, NULL);*/ + }else{ + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + static const int MIN_COMMIT_COUNT = 1; + + int msg_count = 0; + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + return; + } + + tmq_list_t* subList = NULL; + tmq_subscription(tmq, &subList); + char** subTopics = tmq_list_to_c_array(subList); + int32_t sz = tmq_list_get_size(subList); + printf("subscribed topics: "); + for (int32_t i = 0; i < sz; i++) { + printf("%s, ", subTopics[i]); + } + printf("\n"); + tmq_list_destroy(subList); + + while (running) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 1000); + if (tmqmessage) { + msg_process(tmqmessage); + taos_free_result(tmqmessage); + + /*tmq_commit_sync(tmq, NULL);*/ + /*if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);*/ + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void initLogFile() { + char f1[256] = {0}; + char f2[256] = {0}; + + sprintf(f1, "%s/../log/tmq_taosx_tmp.source", dir); + sprintf(f2, "%s/../log/tmq_taosx_tmp.result", dir); + TdFilePtr pFile = taosOpenFile(f1, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile) { + fprintf(stderr, "Failed to open %s for save result\n", f1); + exit(-1); + } + g_fp = pFile; + + TdFilePtr pFile2 = taosOpenFile(f2, TD_FILE_TEXT | TD_FILE_TRUNC | TD_FILE_STREAM); + if (NULL == pFile2) { + fprintf(stderr, "Failed to open %s for save result\n", f2); + exit(-1); + } + char *result[] = { + "{\"type\":\"create\",\"tableName\":\"st1\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct0\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}", + "{\"type\":\"create\",\"tableName\":\"ct1\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}]}", + "{\"type\":\"create\",\"tableName\":\"ct2\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[]}", + "{\"type\":\"create\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}]}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableName\":\"st1\",\"tableType\":\"super\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableName\":\"ct3\",\"tableType\":\"child\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}", + "{\"type\":\"create\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":9}", + "{\"type\":\"alter\",\"tableName\":\"n1\",\"tableType\":\"normal\",\"alterType\":6,\"colName\":\"c1\"}", + "{\"type\":\"create\",\"tableName\":\"jt\",\"tableType\":\"super\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableName\":\"jt1\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}]}", + "{\"type\":\"create\",\"tableName\":\"jt2\",\"tableType\":\"child\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]}" + }; + + for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + taosFprintfFile(pFile2, result[i]); + taosFprintfFile(pFile2, "\n"); + } + taosCloseFile(&pFile2); +} + +int main(int argc, char* argv[]) { + if(argc == 3 && strcmp(argv[1], "-c") == 0) { + strcpy(dir, argv[2]); + }else{ + strcpy(dir, "../../../sim/psim/cfg"); + } + + printf("env init\n"); + initLogFile(); + + if (init_env() < 0) { + return -1; + } + create_topic(); + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + /*sync_consume_loop(tmq, topic_list);*/ + taosCloseFile(&g_fp); +} From 58514862e6f30ea847e822bfded0411e440ca2eb Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 27 Jul 2022 15:25:57 +0800 Subject: [PATCH 25/43] feat: delete sink node add ts column name attribute --- include/libs/nodes/plannodes.h | 1 + source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 7 +++++++ source/libs/planner/src/planLogicCreater.c | 1 + source/libs/planner/src/planPhysiCreater.c | 1 + 5 files changed, 11 insertions(+) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 1147e875f2..0e05ed7887 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -152,6 +152,7 @@ typedef struct SVnodeModifyLogicNode { uint64_t stableId; int8_t tableType; // table type char tableFName[TSDB_TABLE_FNAME_LEN]; + char tsColName[TSDB_COL_NAME_LEN]; STimeWindow deleteTimeRange; SVgroupsInfo* pVgroupList; SNodeList* pInsertCols; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 5279d015b4..edae8ae118 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -402,6 +402,7 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi COPY_SCALAR_FIELD(stableId); COPY_SCALAR_FIELD(tableType); COPY_CHAR_ARRAY_FIELD(tableFName); + COPY_CHAR_ARRAY_FIELD(tsColName); COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow)); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); CLONE_NODE_LIST_FIELD(pInsertCols); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 186a51f000..51bd2c644b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2370,6 +2370,7 @@ static int32_t jsonToPhysiQueryInsertNode(const SJson* pJson, void* pObj) { static const char* jkDeletePhysiPlanTableId = "TableId"; static const char* jkDeletePhysiPlanTableType = "TableType"; static const char* jkDeletePhysiPlanTableFName = "TableFName"; +static const char* jkDeletePhysiPlanTsColName = "TsColName"; static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey"; static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey"; static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows"; @@ -2387,6 +2388,9 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddStringToObject(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDeletePhysiPlanTsColName, pNode->tsColName); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, pNode->deleteTimeRange.skey); } @@ -2413,6 +2417,9 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkDeletePhysiPlanTableFName, pNode->tableFName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDeletePhysiPlanTsColName, pNode->tsColName); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkDeletePhysiPlanDeleteTimeRangeStartKey, &pNode->deleteTimeRange.skey); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 30e3b676df..8f8d666bd3 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1294,6 +1294,7 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet pModify->tableType = pRealTable->pMeta->tableType; snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName); + strcpy(pModify->tsColName, pRealTable->pMeta->schema->name); pModify->deleteTimeRange = pDelete->timeRange; pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc); if (NULL == pModify->pAffectedRows) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 587e566939..402989024c 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1637,6 +1637,7 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pDeleter->tableId = pModify->tableId; pDeleter->tableType = pModify->tableType; strcpy(pDeleter->tableFName, pModify->tableFName); + strcpy(pDeleter->tsColName, pModify->tsColName); pDeleter->deleteTimeRange = pModify->deleteTimeRange; int32_t code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows, From 75974255c7e50c6f75b07922318d7a9f8bafb0f7 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 27 Jul 2022 16:26:41 +0800 Subject: [PATCH 26/43] fix: fix mnd meta issue --- source/dnode/mnode/impl/src/mndQuery.c | 8 +++++--- source/libs/catalog/src/ctgRemote.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index e6e9cae100..2beeb10335 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -105,12 +105,14 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; return -1; } - - code = (*fp)(&reqMsg); + if ((*fp)(&reqMsg)) { + rsp.rspCode = terrno; + } else { + rsp.rspCode = 0; + } rsp.reqType = reqMsg.msgType; rsp.msgLen = reqMsg.info.rspLen; - rsp.rspCode = code; rsp.msg = reqMsg.info.rsp; taosArrayPush(batchRsp, &rsp); diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 3ec30ac39f..55bfc88a49 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -27,7 +27,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu int32_t taskNum = taosArrayGetSize(pTaskId); SDataBuf taskMsg = *pMsg; int32_t offset = 0; - int32_t msgNum = (pMsg->pData && (pMsg->len > 0)) ? ntohl(*(int32_t*)pMsg->pData) : 0; + int32_t msgNum = (TSDB_CODE_SUCCESS == rspCode && pMsg->pData && (pMsg->len > 0)) ? ntohl(*(int32_t*)pMsg->pData) : 0; ASSERT(taskNum == msgNum || 0 == msgNum); ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); From 00acf4520c3418350afd962607fbd6715ca6758e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 27 Jul 2022 16:49:03 +0800 Subject: [PATCH 27/43] refactor(stream): remove option --- include/libs/stream/tstream.h | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 2 +- source/dnode/mnode/impl/src/mndScheduler.c | 2 ++ source/dnode/vnode/src/tq/tq.c | 2 +- source/libs/stream/src/streamTask.c | 4 ++-- source/libs/wal/inc/walInt.h | 6 ++++++ 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6199732cc8..ab1c00a694 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -270,7 +270,7 @@ typedef struct SStreamTask { int64_t startVer; int64_t checkpointVer; int64_t processedVer; - int32_t numOfVgroups; + // int32_t numOfVgroups; // children info SArray* childEpInfo; // SArray diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 5ad13e383a..9d1142801d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -89,7 +89,7 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to stream since %s", pVnode->vgId, pMsg, terrstr()); + dGError("vgId:%d, msg:%p failed to process stream since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 9882b0a9ae..9d7fa537bb 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -391,10 +391,12 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { // exec pInnerTask->execType = TASK_EXEC__PIPE; +#if 0 SDbObj* pSourceDb = mndAcquireDb(pMnode, pStream->sourceDb); ASSERT(pDbObj != NULL); sdbRelease(pSdb, pSourceDb); pInnerTask->numOfVgroups = pSourceDb->cfg.numOfVgroups; +#endif if (tsSchedStreamToSnode) { SSnodeObj* pSnode = mndSchedFetchOneSnode(pMnode); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 01f2f659ff..6b0e3944e3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -653,7 +653,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { } else { SReadHandle mgHandle = { .vnode = NULL, - .numOfVgroups = pTask->numOfVgroups, + .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 5921e44a9c..216e3fa761 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -64,7 +64,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->numOfVgroups) < 0) return -1; + /*if (tEncodeI32(pEncoder, pTask->numOfVgroups) < 0) return -1;*/ int32_t epSz = taosArrayGetSize(pTask->childEpInfo); if (tEncodeI32(pEncoder, epSz) < 0) return -1; @@ -119,7 +119,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->numOfVgroups) < 0) return -1; + /*if (tDecodeI32(pDecoder, &pTask->numOfVgroups) < 0) return -1;*/ int32_t epSz; if (tDecodeI32(pDecoder, &epSz) < 0) return -1; diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 20667fc918..3aebb1c6ba 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -61,26 +61,31 @@ static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) } static inline int64_t walGetLastFileSize(SWal* pWal) { + if (taosArrayGetSize(pWal->fileInfoSet) == 0) return 0; SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); return pInfo->fileSize; } static inline int64_t walGetLastFileFirstVer(SWal* pWal) { + if (taosArrayGetSize(pWal->fileInfoSet) == 0) return -1; SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); return pInfo->firstVer; } static inline int64_t walGetCurFileFirstVer(SWal* pWal) { + if (pWal->writeCur == -1) return -1; SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->firstVer; } static inline int64_t walGetCurFileLastVer(SWal* pWal) { + if (pWal->writeCur == -1) return -1; SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->firstVer; } static inline int64_t walGetCurFileOffset(SWal* pWal) { + if (pWal->writeCur == -1) return -1; SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->fileSize; } @@ -88,6 +93,7 @@ static inline int64_t walGetCurFileOffset(SWal* pWal) { static inline bool walCurFileClosed(SWal* pWal) { return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; } static inline SWalFileInfo* walGetCurFileInfo(SWal* pWal) { + if (pWal->writeCur == -1) return NULL; return (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); } From 5e1bbe0e7e0474338b2b5a968150434ac96d8cbf Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 27 Jul 2022 16:55:23 +0800 Subject: [PATCH 28/43] refactor(sma): remove option --- include/libs/executor/executor.h | 7 +++---- source/dnode/vnode/src/sma/smaRollup.c | 10 +++++----- source/libs/executor/src/executor.c | 18 ++++++++---------- source/libs/stream/src/streamExec.c | 8 ++++---- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 65e20336cc..500418df97 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -64,8 +64,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); * @param SReadHandle * @return */ -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, - SSchemaWrapper** pSchema); +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema); /** * Set the input data block for the stream scan. @@ -74,7 +73,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n * @param type * @return */ -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid); +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type); /** * Set multiple input data blocks for the stream scan. @@ -84,7 +83,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool * @param type * @return */ -int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type, bool assignUid); +int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type); /** * Update the table id list, add or remove. diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a6fde1e2d2..cf489d72a8 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -611,8 +611,8 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche goto _err; } - smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " version:%"PRIi64, SMA_VID(pSma), - suid, pItem->level, output->info.version); + smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " version:%" PRIi64, + SMA_VID(pSma), suid, pItem->level, output->info.version); taosMemoryFreeClear(pReq); taosArrayClear(pResult); @@ -644,7 +644,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, pItem->taskInfo, suid); - if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // INPUT__DATA_SUBMIT + if (qSetStreamInput(pItem->taskInfo, pMsg, inputType) < 0) { // INPUT__DATA_SUBMIT smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } @@ -1329,7 +1329,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { tdRefRSmaInfo(pSma, pRSmaInfo); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); + qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK); tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); tdUnRefRSmaInfo(pSma, pRSmaInfo); @@ -1356,4 +1356,4 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { _end: tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); -} \ No newline at end of file +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 8b1cbb5ae8..5d4e5c9e8d 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -30,8 +30,7 @@ static void cleanupRefPool() { taosCloseRef(ref); } -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid, - char* id) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { @@ -44,12 +43,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu return TSDB_CODE_QRY_APP_ERROR; } pOperator->status = OP_NOT_OPENED; - return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, assignUid, id); + return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); } else { pOperator->status = OP_NOT_OPENED; SStreamScanInfo* pInfo = pOperator->info; - pInfo->assignBlockUid = assignUid; + /*pInfo->assignBlockUid = assignUid;*/ // TODO: if a block was set but not consumed, // prevent setting a different type of block @@ -95,11 +94,11 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { - return qSetMultiStreamInput(tinfo, input, 1, type, assignUid); +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type) { + return qSetMultiStreamInput(tinfo, input, 1, type); } -int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type, bool assignUid) { +int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { return TSDB_CODE_QRY_APP_ERROR; } @@ -110,8 +109,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t code = - doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, assignUid, GET_TASKID(pTaskInfo)); + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { @@ -337,7 +335,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pSinkParam); } } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f782de95b9..2b2c96472d 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -22,22 +22,22 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) SStreamQueueItem* pItem = (SStreamQueueItem*)data; if (pItem->type == STREAM_INPUT__GET_RES) { SStreamTrigger* pTrigger = (SStreamTrigger*)data; - qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK, false); + qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->isDataScan); SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); - qSetStreamInput(exec, pSubmit->data, STREAM_INPUT__DATA_SUBMIT, false); + qSetStreamInput(exec, pSubmit->data, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SStreamDataBlock* pBlock = (SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; qDebug("task %d %p set ssdata input", pTask->taskId, pTask); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK, false); + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)data; SArray* blocks = pMerged->reqs; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT, false); + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); } else { ASSERT(0); } From 2ba6ac5fbafef4daa8a95c20b756e59cf9674db9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 27 Jul 2022 17:11:57 +0800 Subject: [PATCH 29/43] feat:add ts,table name from SDeleterRes --- include/common/tmsg.h | 3 +- include/libs/executor/dataSinkMgt.h | 3 +- include/libs/nodes/plannodes.h | 6 +-- source/client/src/tmq.c | 62 +++++++++++----------- source/common/src/tmsg.c | 2 + source/libs/executor/src/dataDeleter.c | 3 +- source/libs/nodes/src/nodesCloneFuncs.c | 2 +- source/libs/nodes/src/nodesCodeFuncs.c | 4 +- source/libs/planner/src/planLogicCreater.c | 6 +-- source/libs/planner/src/planPhysiCreater.c | 4 +- source/libs/qworker/src/qworker.c | 3 +- 11 files changed, 51 insertions(+), 47 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 73238697cb..563d6c766c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3044,7 +3044,8 @@ typedef struct SDeleteRes { int64_t skey; int64_t ekey; int64_t affectedRows; - char tableFName[TSDB_TABLE_FNAME_LEN]; + char tableFName[TSDB_TABLE_NAME_LEN]; + char tsColName[TSDB_COL_NAME_LEN]; } SDeleteRes; int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 90b804b382..47177dc11b 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -38,7 +38,8 @@ typedef struct SDeleterRes { int64_t skey; int64_t ekey; int64_t affectedRows; - char tableFName[TSDB_TABLE_FNAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + char tsColName[TSDB_COL_NAME_LEN]; } SDeleterRes; typedef struct SDeleterParam { diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 0e05ed7887..9fe9760bb0 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -151,7 +151,7 @@ typedef struct SVnodeModifyLogicNode { uint64_t tableId; uint64_t stableId; int8_t tableType; // table type - char tableFName[TSDB_TABLE_FNAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN]; STimeWindow deleteTimeRange; SVgroupsInfo* pVgroupList; @@ -494,7 +494,7 @@ typedef struct SQueryInserterNode { uint64_t tableId; uint64_t stableId; int8_t tableType; // table type - char tableFName[TSDB_TABLE_FNAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; int32_t vgId; SEpSet epSet; } SQueryInserterNode; @@ -503,7 +503,7 @@ typedef struct SDataDeleterNode { SDataSinkNode sink; uint64_t tableId; int8_t tableType; // table type - char tableFName[TSDB_TABLE_FNAME_LEN]; + char tableFName[TSDB_TABLE_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN]; STimeWindow deleteTimeRange; SNode* pAffectedRows; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index df9072fe1a..88ebb099e5 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -2823,35 +2823,35 @@ end: // delete from db.tabl where .. -> delete from tabl where .. // delete from db .tabl where .. -> delete from tabl where .. -static void getTbName(char *sql){ - char *ch = sql; - - bool inBackQuote = false; - int8_t dotIndex = 0; - while(*ch != '\0'){ - if(!inBackQuote && *ch == '`'){ - inBackQuote = true; - ch++; - continue; - } - - if(inBackQuote && *ch == '`'){ - inBackQuote = false; - ch++; - - continue; - } - - if(!inBackQuote && *ch == '.'){ - dotIndex ++; - if(dotIndex == 2){ - memmove(sql, ch + 1, strlen(ch + 1) + 1); - break; - } - } - ch++; - } -} +//static void getTbName(char *sql){ +// char *ch = sql; +// +// bool inBackQuote = false; +// int8_t dotIndex = 0; +// while(*ch != '\0'){ +// if(!inBackQuote && *ch == '`'){ +// inBackQuote = true; +// ch++; +// continue; +// } +// +// if(inBackQuote && *ch == '`'){ +// inBackQuote = false; +// ch++; +// +// continue; +// } +// +// if(!inBackQuote && *ch == '.'){ +// dotIndex ++; +// if(dotIndex == 2){ +// memmove(sql, ch + 1, strlen(ch + 1) + 1); +// break; +// } +// } +// ch++; +// } +//} static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { SDeleteRes req = {0}; @@ -2867,9 +2867,9 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - getTbName(req.tableFName); +// getTbName(req.tableFName); char sql[256] = {0}; - sprintf(sql, "delete from `%s` where `%s` >= %" PRId64" and `%s` <= %" PRId64, req.tableFName, "ts", req.skey, "ts", req.ekey); + sprintf(sql, "delete from `%s` where `%s` >= %" PRId64" and `%s` <= %" PRId64, req.tableFName, req.tsColName, req.skey, req.tsColName, req.ekey); printf("delete sql:%s\n", sql); TAOS_RES* res = taos_query(taos, sql); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 59d3b45e09..68d7a3d292 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5682,6 +5682,7 @@ int32_t tEncodeDeleteRes(SEncoder *pCoder, const SDeleteRes *pRes) { if (tEncodeI64v(pCoder, pRes->affectedRows) < 0) return -1; if (tEncodeCStr(pCoder, pRes->tableFName) < 0) return -1; + if (tEncodeCStr(pCoder, pRes->tsColName) < 0) return -1; return 0; } @@ -5700,6 +5701,7 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { if (tDecodeI64v(pCoder, &pRes->affectedRows) < 0) return -1; if (tDecodeCStrTo(pCoder, pRes->tableFName) < 0) return -1; + if (tDecodeCStrTo(pCoder, pRes->tsColName) < 0) return -1; return 0; } int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 391aef529f..06b7c13fa2 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -90,7 +90,8 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pRes->uidList = pHandle->pParam->pUidList; pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; - strcpy(pRes->tableFName, pHandle->pDeleter->tableFName); + strcpy(pRes->tableName, pHandle->pDeleter->tableFName); + strcpy(pRes->tsColName, pHandle->pDeleter->tsColName); pRes->affectedRows = *(int64_t*)pColRes->pData; pBuf->useSize += pEntry->dataLen; diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index edae8ae118..5fc94c2642 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -401,7 +401,7 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi COPY_SCALAR_FIELD(tableId); COPY_SCALAR_FIELD(stableId); COPY_SCALAR_FIELD(tableType); - COPY_CHAR_ARRAY_FIELD(tableFName); + COPY_CHAR_ARRAY_FIELD(tableName); COPY_CHAR_ARRAY_FIELD(tsColName); COPY_OBJECT_FIELD(deleteTimeRange, sizeof(STimeWindow)); CLONE_OBJECT_FIELD(pVgroupList, vgroupsInfoClone); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 51bd2c644b..d25b2f3e25 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2326,7 +2326,7 @@ static int32_t physiQueryInsertNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkQueryInsertPhysiPlanTableType, pNode->tableType); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkQueryInsertPhysiPlanTableFName, pNode->tableFName); + code = tjsonAddStringToObject(pJson, jkQueryInsertPhysiPlanTableFName, pNode->tableName); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkQueryInsertPhysiPlanVgId, pNode->vgId); @@ -2355,7 +2355,7 @@ static int32_t jsonToPhysiQueryInsertNode(const SJson* pJson, void* pObj) { code = tjsonGetTinyIntValue(pJson, jkQueryInsertPhysiPlanTableType, &pNode->tableType); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkQueryInsertPhysiPlanTableFName, pNode->tableFName); + code = tjsonGetStringValue(pJson, jkQueryInsertPhysiPlanTableFName, pNode->tableName); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkQueryInsertPhysiPlanVgId, &pNode->vgId); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 8f8d666bd3..d405b75003 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1292,8 +1292,7 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet pModify->modifyType = MODIFY_TABLE_TYPE_DELETE; pModify->tableId = pRealTable->pMeta->uid; pModify->tableType = pRealTable->pMeta->tableType; - snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId, - pRealTable->table.dbName, pRealTable->table.tableName); + snprintf(pModify->tableName, sizeof(pModify->tableName), "%s", pRealTable->table.tableName); strcpy(pModify->tsColName, pRealTable->pMeta->schema->name); pModify->deleteTimeRange = pDelete->timeRange; pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc); @@ -1343,8 +1342,7 @@ static int32_t createVnodeModifLogicNodeByInsert(SLogicPlanContext* pCxt, SInser pModify->tableId = pRealTable->pMeta->uid; pModify->stableId = pRealTable->pMeta->suid; pModify->tableType = pRealTable->pMeta->tableType; - snprintf(pModify->tableFName, sizeof(pModify->tableFName), "%d.%s.%s", pCxt->pPlanCxt->acctId, - pRealTable->table.dbName, pRealTable->table.tableName); + snprintf(pModify->tableName, sizeof(pModify->tableName), "%s", pRealTable->table.tableName); TSWAP(pModify->pVgroupList, pRealTable->pVgroupList); pModify->pInsertCols = nodesCloneList(pInsert->pCols); if (NULL == pModify->pInsertCols) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 402989024c..beb3dd0067 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1586,7 +1586,7 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod pInserter->tableId = pModify->tableId; pInserter->stableId = pModify->stableId; pInserter->tableType = pModify->tableType; - strcpy(pInserter->tableFName, pModify->tableFName); + strcpy(pInserter->tableName, pModify->tableName); pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId; pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet; vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode); @@ -1636,7 +1636,7 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* pDeleter->tableId = pModify->tableId; pDeleter->tableType = pModify->tableType; - strcpy(pDeleter->tableFName, pModify->tableFName); + strcpy(pDeleter->tableFName, pModify->tableName); strcpy(pDeleter->tsColName, pModify->tsColName); pDeleter->deleteTimeRange = pModify->deleteTimeRange; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index df57d0fef1..d1f8a50dab 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -283,7 +283,8 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes pRes->skey = pDelRes->skey; pRes->ekey = pDelRes->ekey; pRes->affectedRows = pDelRes->affectedRows; - strcpy(pRes->tableFName, pDelRes->tableFName); + strcpy(pRes->tableFName, pDelRes->tableName); + strcpy(pRes->tsColName, pDelRes->tsColName); taosMemoryFree(output.pData); return TSDB_CODE_SUCCESS; From 40e67b676cc755a4b8a7f40df9854b4125fd5937 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Jul 2022 19:32:50 +0800 Subject: [PATCH 30/43] test: valgrind case --- tests/script/jenkins/basic.txt | 2 +- tests/script/tsim/stable/show.sim | 61 ------------------------------- 2 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 tests/script/tsim/stable/show.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index f34f29fb64..8de606e705 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -274,7 +274,6 @@ ./test.sh -f tsim/stable/dnode3.sim ./test.sh -f tsim/stable/metrics.sim ./test.sh -f tsim/stable/refcount.sim -./test.sh -f tsim/stable/show.sim ./test.sh -f tsim/stable/tag_add.sim ./test.sh -f tsim/stable/tag_drop.sim ./test.sh -f tsim/stable/tag_filter.sim @@ -308,6 +307,7 @@ ./test.sh -f tsim/valgrind/checkError4.sim ./test.sh -f tsim/valgrind/checkError5.sim ./test.sh -f tsim/valgrind/checkError6.sim +./test.sh -f tsim/valgrind/checkError7.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim diff --git a/tests/script/tsim/stable/show.sim b/tests/script/tsim/stable/show.sim deleted file mode 100644 index d3ab75adf5..0000000000 --- a/tests/script/tsim/stable/show.sim +++ /dev/null @@ -1,61 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -print ======================== create stable -sql create database d1 -sql use d1 - -$x = 0 -while $x < 128 - $tb = d1.s . $x - sql create table $tb (ts timestamp, i int) tags (j int) - $x = $x + 1 -endw - -print ======================== describe stables -# TODO : create stable error -$m = 0 -while $m < 128 - $tb = s . $m - $filter = ' . $tb - $filter = $filter . ' - sql show stables like $filter - print sql : show stables like $filter - if $rows != 1 then - print expect 1, actual: $rows - return -1 - endi - $m = $m + 1 -endw - - -print ======================== show stables - -sql show d1.stables - -print num of stables is $rows -if $rows != 128 then - return -1 -endi - -print ======================== create table - -$x = 0 -while $x < 424 - $tb = d1.t . $x - sql create table $tb using d1.s0 tags( $x ) - $x = $x + 1 -endw - -print ======================== show stables - -sql show d1.tables - -print num of tables is $rows -if $rows != 424 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT From ba0c1b368907fe798f63ee4ad7f3bba08985a8e4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 27 Jul 2022 20:04:31 +0800 Subject: [PATCH 31/43] refactor(stream) --- include/libs/executor/executor.h | 9 --------- source/dnode/vnode/src/sma/smaRollup.c | 4 ++-- source/libs/executor/src/executor.c | 4 ---- source/libs/stream/src/streamDispatch.c | 20 ++++++++++++++++---- source/libs/stream/src/streamExec.c | 2 +- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 500418df97..a7fae403ed 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -66,15 +66,6 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); */ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema); -/** - * Set the input data block for the stream scan. - * @param tinfo - * @param input - * @param type - * @return - */ -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type); - /** * Set multiple input data blocks for the stream scan. * @param tinfo diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index cf489d72a8..4b29b13abd 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -644,7 +644,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, pItem->taskInfo, suid); - if (qSetStreamInput(pItem->taskInfo, pMsg, inputType) < 0) { // INPUT__DATA_SUBMIT + if (qSetMultiStreamInput(pItem->taskInfo, pMsg, 1, inputType) < 0) { // INPUT__DATA_SUBMIT smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } @@ -1329,7 +1329,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { tdRefRSmaInfo(pSma, pRSmaInfo); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK); + qSetMultiStreamInput(pItem->taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK); tdRSmaFetchAndSubmitResult(pItem, pRSmaInfo->pTSchema, pRSmaInfo->suid, pStat, STREAM_INPUT__DATA_BLOCK); tdUnRefRSmaInfo(pSma, pRSmaInfo); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 5d4e5c9e8d..1618bffb09 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -94,10 +94,6 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type) { - return qSetMultiStreamInput(tinfo, input, 1, type); -} - int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { return TSDB_CODE_QRY_APP_ERROR; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 2e33632f12..44f38823ee 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -227,6 +227,8 @@ int32_t streamDispatchOneReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, msg.pCont = buf; msg.msgType = pTask->dispatchMsgType; + qDebug("dispatch from task %d to task %d node %d", pTask->taskId, pReq->taskId, vgId); + tmsgSendReq(pEpSet, &msg); code = 0; @@ -281,8 +283,10 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat return code; } else if (pTask->dispatchType == TASK_DISPATCH__SHUFFLE) { - SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; - ASSERT(pTask->shuffleDispatcher.waitingRspCnt == 0); + int32_t rspCnt = atomic_load_32(&pTask->shuffleDispatcher.waitingRspCnt); + ASSERT(rspCnt == 0); + + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t vgSz = taosArrayGetSize(vgInfo); SStreamDispatchReq* pReqs = taosMemoryCalloc(vgSz, sizeof(SStreamDispatchReq)); if (pReqs == NULL) { @@ -301,7 +305,10 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (pReqs[i].data == NULL || pReqs[i].dataLen == NULL) { goto FAIL_SHUFFLE_DISPATCH; } + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + pReqs[i].taskId = pVgInfo->taskId; } + for (int32_t i = 0; i < blockNum; i++) { SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pDataBlock->info.groupId); @@ -309,6 +316,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat // TODO: get hash function by hashMethod uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); + taosMemoryFree(ctbName); + + bool found = false; // TODO: optimize search int32_t j; for (j = 0; j < vgSz; j++) { @@ -318,12 +328,14 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { goto FAIL_SHUFFLE_DISPATCH; } - pReqs[j].taskId = pVgInfo->taskId; pReqs[j].blockNum++; + found = true; break; } } + ASSERT(found); } + for (int32_t i = 0; i < vgSz; i++) { if (pReqs[i].blockNum > 0) { // send @@ -331,7 +343,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (streamDispatchOneReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { goto FAIL_SHUFFLE_DISPATCH; } - pTask->shuffleDispatcher.waitingRspCnt++; + atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } } code = 0; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 2b2c96472d..7c5cd6e391 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -27,7 +27,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) ASSERT(pTask->isDataScan); SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); - qSetStreamInput(exec, pSubmit->data, STREAM_INPUT__DATA_SUBMIT); + qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SStreamDataBlock* pBlock = (SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; From e2648a7023c2a28fa447d09816146e70c00c79f1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Jul 2022 20:29:25 +0800 Subject: [PATCH 32/43] test: restore case --- tests/script/jenkins/basic.txt | 16 ++++----- tests/script/tsim/parser/limit1_stb.sim | 47 +------------------------ 2 files changed, 9 insertions(+), 54 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 8de606e705..9be5f4fe59 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,7 +1,7 @@ #======================b1-start=============== -# ---- user +# ---- user ---- ./test.sh -f tsim/user/basic.sim ./test.sh -f tsim/user/password.sim ./test.sh -f tsim/user/privilege_db.sim @@ -58,13 +58,13 @@ # unsupport ./test.sh -f tsim/dnode/vnode_clean.sim ./test.sh -f tsim/dnode/use_dropped_dnode.sim -# ---- import +# ---- import ---- ./test.sh -f tsim/import/basic.sim ./test.sh -f tsim/import/commit.sim ./test.sh -f tsim/import/large.sim ./test.sh -f tsim/import/replica1.sim -# ---- insert +# ---- insert ---- ./test.sh -f tsim/insert/backquote.sim ./test.sh -f tsim/insert/basic.sim ./test.sh -f tsim/insert/basic0.sim @@ -164,7 +164,7 @@ # TD-17704 ./test.sh -f tsim/parser/union_sysinfo.sim # TD-17661 ./test.sh -f tsim/parser/where.sim -# ---- query +# ---- query ---- ./test.sh -f tsim/query/charScalarFunction.sim # ./test.sh -f tsim/query/explain.sim ./test.sh -f tsim/query/interval-offset.sim @@ -190,7 +190,7 @@ ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim -# ---- show +# ---- show ---- ./test.sh -f tsim/show/basic.sim # ---- table @@ -260,7 +260,7 @@ ./test.sh -f tsim/tmq/snapshot.sim ./test.sh -f tsim/tmq/snapshot1.sim -# --- stable +# --- stable ---- ./test.sh -f tsim/stable/alter_comment.sim ./test.sh -f tsim/stable/alter_count.sim ./test.sh -f tsim/stable/alter_import.sim @@ -350,13 +350,13 @@ ./test.sh -f tsim/column/metrics.sim ./test.sh -f tsim/column/table.sim -# ---- compress +# ---- compress ---- ./test.sh -f tsim/compress/commitlog.sim ./test.sh -f tsim/compress/compress2.sim ./test.sh -f tsim/compress/compress.sim ./test.sh -f tsim/compress/uncompress.sim -# ---- compute +# ---- compute ---- ./test.sh -f tsim/compute/avg.sim ./test.sh -f tsim/compute/block_dist.sim ./test.sh -f tsim/compute/bottom.sim diff --git a/tests/script/tsim/parser/limit1_stb.sim b/tests/script/tsim/parser/limit1_stb.sim index 7745e2d032..43fa06230c 100644 --- a/tests/script/tsim/parser/limit1_stb.sim +++ b/tests/script/tsim/parser/limit1_stb.sim @@ -51,40 +51,10 @@ endi ##TBASE-352 $offset = $tbNum * $rowNum $offset = $offset - 1 -sql select * from $stb limit 2 offset $offset +sql select * from $stb order by ts limit 2 offset $offset if $rows != 1 then return -1 endi -if $data00 != @18-11-25 19:30:00.000@ then - return -1 -endi -if $data01 != 9 then - return -1 -endi -if $data02 != NULL then - return -1 -endi -if $data03 != 9.00000 then - return -1 -endi -if $data04 != NULL then - return -1 -endi -if $data05 != 9 then - return -1 -endi -if $data06 != 9 then - return -1 -endi -if $data07 != 1 then - return -1 -endi -if $data08 != binary9 then - return -1 -endi -if $data09 != nchar9 then - return -1 -endi $offset = $tbNum * $rowNum $offset = $offset / 2 @@ -207,21 +177,6 @@ sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 if $rows != 5 then return -1 endi -if $data01 != 5 then - return -1 -endi -if $data11 != 6 then - return -1 -endi -if $data21 != 7 then - return -1 -endi -if $data31 != 8 then - return -1 -endi -if $data41 != 4 then - return -1 -endi $offset = $totalNum / 2 sql select * from $stb where ts >= $ts0 and ts <= $tsu limit 5 offset $offset From 904ec81bf96f3f65d7e99fccd577e82a3cf3ba53 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 27 Jul 2022 21:02:21 +0800 Subject: [PATCH 33/43] fix(stream): concurrency dispatch --- source/libs/stream/src/streamDispatch.c | 4 +++- tests/script/jenkins/basic.txt | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 44f38823ee..834a3af0d5 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -328,6 +328,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { goto FAIL_SHUFFLE_DISPATCH; } + if (pReqs[j].blockNum == 0) { + atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); + } pReqs[j].blockNum++; found = true; break; @@ -343,7 +346,6 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (streamDispatchOneReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { goto FAIL_SHUFFLE_DISPATCH; } - atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } } code = 0; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 99beabe9ee..5d7aa8eda1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -298,8 +298,9 @@ # --- sma ./test.sh -f tsim/sma/drop_sma.sim ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim -./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +# temp disable +#./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +#./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim # --- valgrind ./test.sh -f tsim/valgrind/checkError1.sim From 28bb8957227f7077caa3c64e87637a1fa508258f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Jul 2022 21:32:18 +0800 Subject: [PATCH 34/43] test: valgrind case --- tests/script/jenkins/basic.txt | 8 ++++---- tests/script/tsim/alter/cached_schema_after_alter.sim | 3 --- tests/script/tsim/query/udf.sim | 4 ---- tests/script/tsim/valgrind/checkError6.sim | 7 +++++-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9be5f4fe59..c0169c7912 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -328,24 +328,24 @@ ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim -# --- catalog +# --- catalog ---- ./test.sh -f tsim/catalog/alterInCurrent.sim # --- scalar ./test.sh -f tsim/scalar/in.sim ./test.sh -f tsim/scalar/scalar.sim -# ---- alter +# ---- alter ---- ./test.sh -f tsim/alter/cached_schema_after_alter.sim ./test.sh -f tsim/alter/dnode.sim ./test.sh -f tsim/alter/table.sim -# ---- cache +# ---- cache ---- ./test.sh -f tsim/cache/new_metrics.sim ./test.sh -f tsim/cache/restart_table.sim ./test.sh -f tsim/cache/restart_metrics.sim -# ---- column +# ---- column ---- ./test.sh -f tsim/column/commit.sim ./test.sh -f tsim/column/metrics.sim ./test.sh -f tsim/column/table.sim diff --git a/tests/script/tsim/alter/cached_schema_after_alter.sim b/tests/script/tsim/alter/cached_schema_after_alter.sim index 043f360856..bd2b1d272c 100644 --- a/tests/script/tsim/alter/cached_schema_after_alter.sim +++ b/tests/script/tsim/alter/cached_schema_after_alter.sim @@ -50,11 +50,8 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -sleep 3000 -sql connect sql use $db sql select * from $stb diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index 4e95095172..5d69887c86 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -1,13 +1,9 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 2 -system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode1 -c udf -v 1 print ========= start dnode1 as leader system sh/exec.sh -n dnode1 -s start -sleep 1000 sql connect print ======== step1 udf diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index 5257468d6d..ec8ca0ad8c 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -63,13 +63,15 @@ sql show table distributed tb1 sql select count(1) from tb1 sql select count(tbcol) as b from tb1 where ts <= 1601481840000 interval(1m) sql select diff(tbcol) from tb1 where ts <= 1601481840000 -sql select diff(tbcol) from tb1 where tbcol > 5 and tbcol < 20 +sql select diff(tbcol) from tb1 where tbcol > 5 and tbcol < 20 order by ts sql select first(tbcol), last(tbcol) as b from tb1 where ts <= 1601481840000 interval(1m) -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from tb1 where ts <= 1601481840000 partition by tgcol interval(1m) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from tb1 where ts <= 1601481840000 partition by tgcol order by tgcol +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from tb1 where ts <= 1601481840000 partition by tgcol interval(1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) sql select last_row(*) from tb1 where tbcol > 5 and tbcol < 20 sql select _wstart, _wend, _wduration, _qstart, _qend, count(*) from tb1 interval(10s, 2s) sliding(10s) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) order by tgcol desc sql select log(tbcol), abs(tbcol), pow(tbcol, 2), sqrt(tbcol), sin(tbcol), cos(tbcol), tan(tbcol), asin(tbcol), acos(tbcol), atan(tbcol), ceil(tbcol), floor(tbcol), round(tbcol), atan(tbcol) from tb1 sql select length("abcd1234"), char_length("abcd1234=-+*") from tb1 sql select tbcol4, length(tbcol4), lower(tbcol4), upper(tbcol4), ltrim(tbcol4), rtrim(tbcol4), concat(tbcol4, tbcol5), concat_ws('_', tbcol4, tbcol5), substr(tbcol4, 1, 4) from tb1 @@ -94,6 +96,7 @@ sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 and sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from stb where ts <= 1601481840000 partition by tgcol interval(1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 and tgcol = 1 partition by tgcol interval(1m) fill(value, 0) order by tgcol desc sql select last_row(tbcol), stddev(tbcol) from stb where tbcol > 5 and tbcol < 20 group by tgcol sql select _wstart, _wend, _wduration, _qstart, _qend, count(*) from stb interval(10s, 2s) sliding(10s) sql select log(tbcol), abs(tbcol), pow(tbcol, 2), sqrt(tbcol), sin(tbcol), cos(tbcol), tan(tbcol), asin(tbcol), acos(tbcol), atan(tbcol), ceil(tbcol), floor(tbcol), round(tbcol), atan(tbcol) from stb From 5b12e7216eeb64d46c6ede4182f07a26450641f3 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 28 Jul 2022 08:58:58 +0800 Subject: [PATCH 35/43] doc: change sidebar label for reference manual --- docs/zh/14-reference/_category_.yml | 2 +- docs/zh/14-reference/index.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/zh/14-reference/_category_.yml b/docs/zh/14-reference/_category_.yml index ae861a15ff..faca32476c 100644 --- a/docs/zh/14-reference/_category_.yml +++ b/docs/zh/14-reference/_category_.yml @@ -1 +1 @@ -label: 参考指南 \ No newline at end of file +label: 参考手册 \ No newline at end of file diff --git a/docs/zh/14-reference/index.md b/docs/zh/14-reference/index.md index a9abff8f42..e9c0c4fe23 100644 --- a/docs/zh/14-reference/index.md +++ b/docs/zh/14-reference/index.md @@ -1,5 +1,4 @@ --- -sidebar_label: 参考手册 title: 参考手册 --- From c81cb676860ca40ee4f8504ae59f9df469b879d4 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 28 Jul 2022 09:54:08 +0800 Subject: [PATCH 36/43] doc: change some documents in operation guide --- docs/zh/12-taos-sql/02-database.md | 2 +- docs/zh/14-reference/14-taosx.md | 4 +++ docs/zh/17-operation/02-planning.mdx | 34 +++++++----------- docs/zh/17-operation/03-tolerance.md | 14 ++++---- docs/zh/17-operation/06-admin.md | 42 ---------------------- docs/zh/17-operation/09-status.md | 53 ---------------------------- 6 files changed, 25 insertions(+), 124 deletions(-) create mode 100644 docs/zh/14-reference/14-taosx.md delete mode 100644 docs/zh/17-operation/06-admin.md delete mode 100644 docs/zh/17-operation/09-status.md diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index edcad98fd8..1349065626 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -46,7 +46,7 @@ database_option: { - last_row：表示缓存子表最近一行数据。这将显著改善 LAST_ROW 函数的性能表现。 - last_value：表示缓存子表每一列的最近的非 NULL 值。这将显著改善无特殊影响（WHERE、ORDER BY、GROUP BY、INTERVAL）下的 LAST 函数的性能表现。 - both：表示同时打开缓存最近行和列功能。 -- CACHESIZE：表示缓存子表最近数据的内存大小。默认为 1 ，范围是[1, 65536]，单位是 MB。 +- CACHESIZE：表示每个 vnode 中用于缓存子表最近数据的内存大小。默认为 1 ，范围是[1, 65536]，单位是 MB。 - COMP：表示数据库文件压缩标志位，缺省值为 2，取值范围为 [0, 2]。 - 0：表示不压缩。 - 1：表示一阶段压缩。 diff --git a/docs/zh/14-reference/14-taosx.md b/docs/zh/14-reference/14-taosx.md new file mode 100644 index 0000000000..ed3f8d488f --- /dev/null +++ b/docs/zh/14-reference/14-taosx.md @@ -0,0 +1,4 @@ +--- +sidebar_label: taosX +title: 使用 taosX 在集群间复制数据 +--- \ No newline at end of file diff --git a/docs/zh/17-operation/02-planning.mdx b/docs/zh/17-operation/02-planning.mdx index 954ba7ca00..9f43ffe561 100644 --- a/docs/zh/17-operation/02-planning.mdx +++ b/docs/zh/17-operation/02-planning.mdx @@ -5,33 +5,25 @@ title: 容量规划 使用 TDengine 来搭建一个物联网大数据平台，计算资源、存储资源需要根据业务场景进行规划。下面分别讨论系统运行所需要的内存、CPU 以及硬盘空间。 -## 内存需求 +## 服务端内存需求 -每个 Database 可以创建固定数目的 vgroup，默认与 CPU 核数相同，可通过 maxVgroupsPerDb 配置；vgroup 中的每个副本会是一个 vnode；每个 vnode 会占用固定大小的内存（大小与数据库的配置参数 blocks 和 cache 有关)；每个 Table 会占用与标签总长度有关的内存；此外，系统会有一些固定的内存开销。因此，每个 DB 需要的系统内存可通过如下公式计算： +每个 Database 可以创建固定数目的 vgroup，默认 2 个 vgroup，在创建数据库时可以通过`vgroups `参数来指定，其副本数由参数`replica `指定。vgroup 中的每个副本会是一个 vnode；所以每个 vnode 占用的内存由以下几个参数决定： -``` -Database Memory Size = maxVgroupsPerDb * replica * (blocks * cache + 10MB) + numOfTables * (tagSizePerTable + 0.5KB) -``` +- vgroups +- replica +- buffer +- pages +- pagesize +- cachesize -示例：假设 maxVgroupPerDB 是缺省值 64，cache 是缺省大小 16M, blocks 是缺省值 6，并且一个 DB 中有 10 万张表，单副本，标签总长度是 256 字节，则这个 DB 总的内存需求为：64 \* 1 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 6792M。 +关于这些参数的详细说明请参考 [数据库管理](../taos-sql/database)。 -在实际的系统运维中，我们通常会更关心 TDengine 服务进程（taosd）会占用的内存量。 +所以一个数据库所需要的内存大小等于 `vgroups * replica * (buffer + pages * pagesize + cachesize)`。 -``` -taosd 内存总量 = vnode 内存 + mnode 内存 + 查询内存 -``` +但要注意的是这些内存并不需要由单一服务器提供，而是由整个集群中所有数据节点共同负担，相当于由这些数据节点所在的服务器共同负担。如果集群中有不止一个数据库，则所需内存要累加，并由集群中所有服务器共同负担。更复杂的场景是如果集群中的数据节点并非在最初就一次性全部建立，而是随着使用中系统负载的增加服务器并增加数据节点，则新创建的数据库会导致新旧数据节点上的负载并不均衡，此时简单的理论计算并不能直接地简单使用，要结合各数据节点的负载情况。 -其中： -1. “vnode 内存”指的是集群中所有的 Database 存储分摊到当前 taosd 节点上所占用的内存资源。可以按上文“Database Memory Size”计算公式估算每个 DB 的内存占用量进行加总，再按集群中总共的 TDengine 节点数做平均（如果设置为多副本，则还需要乘以对应的副本倍数）。 -2. “mnode 内存”指的是集群中管理节点所占用的资源。如果一个 taosd 节点上分布有 mnode 管理节点，则内存消耗还需要增加“0.2KB \* 集群中数据表总数”。 -3. “查询内存”指的是服务端处理查询请求时所需要占用的内存。单条查询语句至少会占用“0.2KB \* 查询涉及的数据表总数”的内存量。 - -注意：以上内存估算方法，主要讲解了系统的“必须内存需求”，而不是“内存总数上限”。在实际运行的生产环境中，由于操作系统缓存、资源管理调度等方面的原因，内存规划应当在估算结果的基础上保留一定冗余，以维持系统状态和系统性能的稳定性。并且，生产环境通常会配置系统资源的监控工具，以便及时发现硬件资源的紧缺情况。 - -最后，如果内存充裕，可以考虑加大 Blocks 的配置，这样更多数据将保存在内存里，提高写入和查询速度。 - -### 客户端内存需求 +## 客户端内存需求 客户端应用采用 taosc 客户端驱动连接服务端，会有内存需求的开销。 @@ -77,5 +69,3 @@ Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable 根据上面的内存、CPU、存储的预估，就可以知道整个系统需要多少核、多少内存、多少存储空间。如果数据副本数不为 1，总需求量需要再乘以副本数。 因为 TDengine 具有很好的水平扩展能力，根据总量，再根据单个物理机或虚拟机的资源，就可以轻松决定需要购置多少台物理机或虚拟机了。 - -**立即计算 CPU、内存、存储，请参见：[资源估算方法](https://www.taosdata.com/config/config.html)。** diff --git a/docs/zh/17-operation/03-tolerance.md b/docs/zh/17-operation/03-tolerance.md index 2c46681962..e2f40778ff 100644 --- a/docs/zh/17-operation/03-tolerance.md +++ b/docs/zh/17-operation/03-tolerance.md @@ -10,19 +10,21 @@ TDengine 接收到应用的请求数据包时，先将请求的原始数据包 涉及的系统配置参数有两个： -- walLevel：WAL 级别，0：不写 WAL; 1：写 WAL, 但不执行 fsync; 2：写 WAL, 而且执行 fsync。 -- fsync：当 walLevel 设置为 2 时，执行 fsync 的周期。设置为 0，表示每次写入，立即执行 fsync。 +- wal_level：WAL 级别，1：写WAL，但不执行fsync。2：写WAL，而且执行fsync。默认值为 1。 +- wal_fsync_period：当 wal_evel 设置为 2 时，执行 fsync 的周期。设置为 0，表示每次写入，立即执行 fsync。 -如果要 100%的保证数据不丢失，需要将 walLevel 设置为 2，fsync 设置为 0。这时写入速度将会下降。但如果应用侧启动的写数据的线程数达到一定的数量(超过 50)，那么写入数据的性能也会很不错，只会比 fsync 设置为 3000 毫秒下降 30%左右。 +如果要 100%的保证数据不丢失，需要将 wal_level 设置为 2，fsync 设置为 0。这时写入速度将会下降。但如果应用侧启动的写数据的线程数达到一定的数量(超过 50)，那么写入数据的性能也会很不错，只会比 fsync 设置为 3000 毫秒下降 30%左右。 ## 灾备 -TDengine 的集群通过多个副本的机制，来提供系统的高可用性，实现灾备能力。 +TDengine 的集群通过多个副本的机制，来提供系统的高可用性，同时具备一定的灾备能力。 -TDengine 集群是由 mnode 负责管理的，为保证 mnode 的高可靠，可以配置多个 mnode 副本，副本数由系统配置参数 numOfMnodes 决定，为了支持高可靠，需要设置大于 1。为保证元数据的强一致性，mnode 副本之间通过同步方式进行数据复制，保证了元数据的强一致性。 +TDengine 集群是由 mnode 负责管理的，为保证 mnode 的高可靠，可以配置 三个 mnode 副本。为保证元数据的强一致性，mnode 副本之间通过同步方式进行数据复制，保证了元数据的强一致性。 -TDengine 集群中的时序数据的副本数是与数据库关联的，一个集群里可以有多个数据库，每个数据库可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数。为了支持高可靠，需要设置副本数大于 1。 +TDengine 集群中的时序数据的副本数是与数据库关联的，一个集群里可以有多个数据库，每个数据库可以配置不同的副本数。创建数据库时，通过参数 replica 指定副本数。为了支持高可靠，需要设置副本数为 3。 TDengine 集群的节点数必须大于等于副本数，否则创建表时将报错。 当 TDengine 集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine 企业版还可以将副本部署在不同机房，从而实现异地容灾。 + +另外一种灾备方式是通过 `taosX` 将一个 TDengine 集群的数据同步复制到物理上位于不同数据中心的另一个 TDengine 集群。其详细使用方法请参考 [taosX 参考手册](../reference/taosX) diff --git a/docs/zh/17-operation/06-admin.md b/docs/zh/17-operation/06-admin.md deleted file mode 100644 index 7934d31eaf..0000000000 --- a/docs/zh/17-operation/06-admin.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: 用户管理 ---- - -系统管理员可以在 CLI 界面里添加、删除用户，也可以修改密码。CLI 里 SQL 语法如下： - -```sql -CREATE USER PASS <'password'>; -``` - -创建用户，并指定用户名和密码，密码需要用单引号引起来，单引号为英文半角 - -```sql -DROP USER ; -``` - -删除用户，限 root 用户使用 - -```sql -ALTER USER PASS <'password'>; -``` - -修改用户密码，为避免被转换为小写，密码需要用单引号引用，单引号为英文半角 - -```sql -ALTER USER PRIVILEGE ; -``` - -修改用户权限为：write 或 read，不需要添加单引号 - -说明：系统内共有 super/write/read 三种权限级别，但目前不允许通过 alter 指令把 super 权限赋予用户。 - -```sql -SHOW USERS; -``` - -显示所有用户 - -:::note -SQL 语法中，< >表示需要用户输入的部分，但请不要输入< >本身。 - -::: diff --git a/docs/zh/17-operation/09-status.md b/docs/zh/17-operation/09-status.md deleted file mode 100644 index e7ae78bace..0000000000 --- a/docs/zh/17-operation/09-status.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: 系统连接、任务查询管理 ---- - -系统管理员可以从 CLI 查询系统的连接、正在进行的查询、流式计算，并且可以关闭连接、停止正在进行的查询和流式计算。 - -## 显示数据库的连接 - -```sql -SHOW CONNECTIONS; -``` - -其结果中的一列显示 ip:port, 为连接的 IP 地址和端口号。 - -## 强制关闭数据库连接 - -```sql -KILL CONNECTION ; -``` - -其中的 connection-id 是 SHOW CONNECTIONS 中显示的第一列的数字。 - -## 显示数据查询 - -```sql -SHOW QUERIES; -``` - -其中第一列显示的以冒号隔开的两个数字为 query-id，为发起该 query 应用连接的 connection-id 和查询次数。 - -## 强制关闭数据查询 - -```sql -KILL QUERY ; -``` - -其中 query-id 是 SHOW QUERIES 中显示的 connection-id:query-no 字串，如“105:2”，拷贝粘贴即可。 - -## 显示连续查询 - -```sql -SHOW STREAMS; -``` - -其中第一列显示的以冒号隔开的两个数字为 stream-id, 为启动该 stream 应用连接的 connection-id 和发起 stream 的次数。 - -## 强制关闭连续查询 - -```sql -KILL STREAM ; -``` - -其中的 stream-id 是 SHOW STREAMS 中显示的 connection-id:stream-no 字串，如 103:2，拷贝粘贴即可。 From 45d389e207be30cd4d22b23e39aefd99a717887f Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Thu, 28 Jul 2022 10:01:47 +0800 Subject: [PATCH 37/43] doc: correct a few paraeters in 02-database.md --- docs/zh/12-taos-sql/02-database.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 1349065626..839b1e2015 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -8,17 +8,17 @@ description: "创建、删除数据库，查看、修改数据库参数" ```sql CREATE DATABASE [IF NOT EXISTS] db_name [database_options] - + database_options: database_option ... - + database_option: { BUFFER value | CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value | COMP {0 | 1 | 2} | DURATION value - | FSYNC value + | WAL_FSYNC_PERIOD value | MAXROWS value | MINROWS value | KEEP value @@ -28,7 +28,7 @@ database_option: { | REPLICA value | RETENTIONS ingestion_duration:keep_duration ... | STRICT {'off' | 'on'} - | WAL {1 | 2} + | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} | WAL_RETENTION_PERIOD value From ee87c7baef5bf02f9937255566e05998a61b42ca Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 28 Jul 2022 10:05:14 +0800 Subject: [PATCH 38/43] Update 02-planning.mdx --- docs/zh/17-operation/02-planning.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/17-operation/02-planning.mdx b/docs/zh/17-operation/02-planning.mdx index 9f43ffe561..06272622cc 100644 --- a/docs/zh/17-operation/02-planning.mdx +++ b/docs/zh/17-operation/02-planning.mdx @@ -7,7 +7,7 @@ title: 容量规划 ## 服务端内存需求 -每个 Database 可以创建固定数目的 vgroup，默认 2 个 vgroup，在创建数据库时可以通过`vgroups `参数来指定，其副本数由参数`replica `指定。vgroup 中的每个副本会是一个 vnode；所以每个 vnode 占用的内存由以下几个参数决定： +每个 Database 可以创建固定数目的 vgroup，默认 2 个 vgroup，在创建数据库时可以通过`vgroups `参数来指定，其副本数由参数`replica `指定。vgroup 中的每个副本会是一个 vnode；所以每个数据库占用的内存由以下几个参数决定： - vgroups - replica @@ -18,7 +18,7 @@ title: 容量规划 关于这些参数的详细说明请参考 [数据库管理](../taos-sql/database)。 -所以一个数据库所需要的内存大小等于 `vgroups * replica * (buffer + pages * pagesize + cachesize)`。 +一个数据库所需要的内存大小等于 `vgroups * replica * (buffer + pages * pagesize + cachesize)`。 但要注意的是这些内存并不需要由单一服务器提供，而是由整个集群中所有数据节点共同负担，相当于由这些数据节点所在的服务器共同负担。如果集群中有不止一个数据库，则所需内存要累加，并由集群中所有服务器共同负担。更复杂的场景是如果集群中的数据节点并非在最初就一次性全部建立，而是随着使用中系统负载的增加服务器并增加数据节点，则新创建的数据库会导致新旧数据节点上的负载并不均衡，此时简单的理论计算并不能直接地简单使用，要结合各数据节点的负载情况。 From 777c40897da30b02566a5130a4e748f5ebbf686a Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 28 Jul 2022 10:19:18 +0800 Subject: [PATCH 39/43] Update 02-planning.mdx --- docs/zh/17-operation/02-planning.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/17-operation/02-planning.mdx b/docs/zh/17-operation/02-planning.mdx index 06272622cc..bf3a4cfdfc 100644 --- a/docs/zh/17-operation/02-planning.mdx +++ b/docs/zh/17-operation/02-planning.mdx @@ -20,7 +20,7 @@ title: 容量规划 一个数据库所需要的内存大小等于 `vgroups * replica * (buffer + pages * pagesize + cachesize)`。 -但要注意的是这些内存并不需要由单一服务器提供，而是由整个集群中所有数据节点共同负担，相当于由这些数据节点所在的服务器共同负担。如果集群中有不止一个数据库，则所需内存要累加，并由集群中所有服务器共同负担。更复杂的场景是如果集群中的数据节点并非在最初就一次性全部建立，而是随着使用中系统负载的增加服务器并增加数据节点，则新创建的数据库会导致新旧数据节点上的负载并不均衡，此时简单的理论计算并不能直接地简单使用，要结合各数据节点的负载情况。 +但要注意的是这些内存并不需要由单一服务器提供，而是由整个集群中所有数据节点共同负担，相当于由这些数据节点所在的服务器共同负担。如果集群中有不止一个数据库，则所需内存要累加，并由集群中所有服务器共同负担。更复杂的场景是如果集群中的数据节点并非在最初就一次性全部建立，而是随着使用中系统负载的增加逐步增加服务器并增加数据节点，则新创建的数据库会导致新旧数据节点上的负载并不均衡，此时简单的理论计算并不能直接使用，要结合各数据节点的负载情况。 ## 客户端内存需求 From 95748e80cfd39492c1bfa13762fca64f3bb35886 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 28 Jul 2022 10:22:45 +0800 Subject: [PATCH 40/43] test: comment out unstable case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 48d60a4adb..7bc3e921f1 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -187,7 +187,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -./test.sh -f tsim/mnode/basic4.sim +# TD-17919 ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim # ---- show ---- From 273ce12df55182feac1dede1990b0e6480b56c5d Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 28 Jul 2022 10:34:07 +0800 Subject: [PATCH 41/43] Update 02-planning.mdx --- docs/zh/17-operation/02-planning.mdx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/zh/17-operation/02-planning.mdx b/docs/zh/17-operation/02-planning.mdx index bf3a4cfdfc..5cf3295288 100644 --- a/docs/zh/17-operation/02-planning.mdx +++ b/docs/zh/17-operation/02-planning.mdx @@ -18,7 +18,11 @@ title: 容量规划 关于这些参数的详细说明请参考 [数据库管理](../taos-sql/database)。 -一个数据库所需要的内存大小等于 `vgroups * replica * (buffer + pages * pagesize + cachesize)`。 +一个数据库所需要的内存大小等于 + +``` +vgroups * replica * (buffer + pages * pagesize + cachesize) +``` 但要注意的是这些内存并不需要由单一服务器提供，而是由整个集群中所有数据节点共同负担，相当于由这些数据节点所在的服务器共同负担。如果集群中有不止一个数据库，则所需内存要累加，并由集群中所有服务器共同负担。更复杂的场景是如果集群中的数据节点并非在最初就一次性全部建立，而是随着使用中系统负载的增加逐步增加服务器并增加数据节点，则新创建的数据库会导致新旧数据节点上的负载并不均衡，此时简单的理论计算并不能直接使用，要结合各数据节点的负载情况。 From 3b47518d451e900708b212eb40af051e8d52a2c7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 28 Jul 2022 10:37:04 +0800 Subject: [PATCH 42/43] test: adjust case --- tests/script/tsim/mnode/basic4.sim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index 0ffcdd8c00..6b4c8c6b7c 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -45,8 +45,11 @@ endi if $data(2)[4] != ready then goto step2 endi +if $data(3)[4] != ready then + goto step2 +endi -system sh/exec.sh -n dnode3 -s stop +system sh/exec.sh -n dnode3 -s stop -x SIGKILL sql_error create mnode on dnode 3 print =============== step3: show mnodes From 40af092385635584ca0f8449c2079e8986c04cb5 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 28 Jul 2022 10:44:21 +0800 Subject: [PATCH 43/43] doc: remove one obsolete parameter in alter dnode --- docs/zh/12-taos-sql/21-node.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/12-taos-sql/21-node.md b/docs/zh/12-taos-sql/21-node.md index 47aa2077a3..4816daf420 100644 --- a/docs/zh/12-taos-sql/21-node.md +++ b/docs/zh/12-taos-sql/21-node.md @@ -40,7 +40,6 @@ ALTER ALL DNODES dnode_option dnode_option: { 'resetLog' - | 'resetQueryCache' | 'balance' value | 'monitor' value | 'debugFlag' value