From 227811f9d53ef26aa4ec8112e0ee16090681b417 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Thu, 30 Jun 2022 09:49:36 +0800 Subject: [PATCH 01/27] update alter_stable case --- tests/system-test/1-insert/alter_stable.py | 129 +++++++++++++++++++-- 1 file changed, 121 insertions(+), 8 deletions(-) diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index cd64e3ddfe..0d82d8fd28 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -16,12 +16,74 @@ import string from util.log import * from util.cases import * from util.sql import * - +from util.sqlset import * +from util import constant +from util.common import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.ntbname = 'ntb' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.binary_length})', + 'col13': f'nchar({self.nchar_length})' + } + self.tag_dict = { + 'ts_tag' : 'timestamp', + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': f'binary({self.binary_length})', + 't13': f'nchar({self.nchar_length})' + } + self.tag_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.tbnum = 1 + self.values_list = [ + f'now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据"' + ] + self.column_add_dict = { + 'col_time' : 'timestamp', + 'col_tinyint' : 'tinyint', + 'col_smallint' : 'smallint', + 'col_int' : 'int', + 'col_bigint' : 'bigint', + 'col_untinyint' : 'tinyint unsigned', + 'col_smallint' : 'smallint unsigned', + 'col_int' : 'int unsigned', + 'col_bigint' : 'bigint unsigned', + 'col_bool' : 'bool', + 'col_float' : 'float', + 'col_double' : 'double', + 'col_binary' : f'binary({constant.BINARY_LENGTH_MAX})', + 'col_nchar' : f'nchar({constant.NCAHR_LENGTH_MAX})' + } def get_long_name(self, length, mode="mixed"): """ generate long name @@ -133,15 +195,66 @@ class TDTestCase: # tdSql.execute(f'create table ntb (ts timestamp,c0 int)') tdSql.error(f'alter stable ntb add column c2 ') tdSql.execute(f'drop database {dbname}') - + def alter_stable_check(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values({j})') + for key,values in self.column_add_dict.items(): + tdSql.execute(f'alter stable {self.stbname} add column {key} {values}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) + tdSql.query(f'select {key} from {self.stbname}_{i}') + tdSql.checkRows(len(self.values_list)) + tdSql.execute(f'alter stable {self.stbname} drop column {key}') + tdSql.query(f'describe {self.stbname}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) + tdSql.error(f'select {key} from {self.stbname} ') + for key,values in self.column_dict.items(): + if 'binary' in values.lower(): + v = f'binary({self.binary_length+1})' + v_error = f'binary({self.binary_length-1})' + tdSql.error(f'alter stable {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter stable {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + result = tdCom.getOneRow(1,'VARCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + elif 'nchar' in values.lower(): + v = f'nchar({self.binary_length+1})' + v_error = f'nchar({self.binary_length-1})' + tdSql.error(f'alter stable {self.stbname} modify column {key} {v_error}') + tdSql.execute(f'alter stable {self.stbname} modify column {key} {v}') + tdSql.query(f'describe {self.stbname}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + for i in range(self.tbnum): + tdSql.query(f'describe {self.stbname}_{i}') + result = tdCom.getOneRow(1,'NCHAR') + tdSql.checkEqual(result[0][2],self.binary_length+1) + else: + for v in self.column_dict.values(): + tdSql.error(f'alter stable {self.stbname} modify column {key} {v}') + pass def run(self): - dbname = self.get_long_name(length=10, mode="letters") - stbname = self.get_long_name(length=5, mode="letters") - tbname = self.get_long_name(length=5, mode="letters") - self.alter_stable_column_check(dbname,stbname,tbname) - self.alter_stable_tag_check(dbname,stbname,tbname) - + # dbname = self.get_long_name(length=10, mode="letters") + # stbname = self.get_long_name(length=5, mode="letters") + # tbname = self.get_long_name(length=5, mode="letters") + # self.alter_stable_column_check(dbname,stbname,tbname) + # self.alter_stable_tag_check(dbname,stbname,tbname) + self.alter_stable_check() def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From c9b3736b8163f6eef891f06d5867cf077b265c76 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Thu, 30 Jun 2022 14:17:13 +0800 Subject: [PATCH 02/27] update test case --- tests/system-test/1-insert/alter_stable.py | 131 +++------------------ tests/system-test/2-query/Now.py | 38 +++--- 2 files changed, 38 insertions(+), 131 deletions(-) diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index 0d82d8fd28..4ff9d2a095 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -84,120 +84,13 @@ class TDTestCase: 'col_nchar' : f'nchar({constant.NCAHR_LENGTH_MAX})' } - def get_long_name(self, length, mode="mixed"): - """ - generate long name - mode could be numbers/letters/letters_mixed/mixed - """ - if mode == "numbers": - population = string.digits - elif mode == "letters": - population = string.ascii_letters.lower() - elif mode == "letters_mixed": - population = string.ascii_letters.upper() + string.ascii_letters.lower() - else: - population = string.ascii_letters.lower() + string.digits - return "".join(random.choices(population, k=length)) - def alter_stable_column_check(self,dbname,stbname,tbname): - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create stable {stbname} (ts timestamp, c1 tinyint, c2 smallint, c3 int, \ - c4 bigint, c5 tinyint unsigned, c6 smallint unsigned, c7 int unsigned, c8 bigint unsigned, c9 float, c10 double, c11 bool,c12 binary(20),c13 nchar(20)) tags(t0 int) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(1)') - tdSql.execute(f'insert into {tbname} values (now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'alter stable {stbname} add column c14 int') - tdSql.query(f'select c14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter stable {stbname} add column `c15` int') - tdSql.query(f'select c15 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter stable {stbname} drop column c14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter stable {stbname} drop column `c15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(15) - tdSql.execute(f'alter stable {stbname} modify column c12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,30) - tdSql.execute(f'alter stable {stbname} modify column `c12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(12,2,35) - tdSql.error(f'alter stable {stbname} modify column `c12` binary(34)') - tdSql.execute(f'alter stable {stbname} modify column c13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(13,2,30) - tdSql.error(f'alter stable {stbname} modify column c13 nchar(29)') - tdSql.error(f'alter stable {stbname} rename column c1 c21') - tdSql.error(f'alter stable {stbname} modify column c1 int') - tdSql.error(f'alter stable {stbname} modify column c4 int') - tdSql.error(f'alter stable {stbname} modify column c8 int') - tdSql.error(f'alter stable {stbname} modify column c1 unsigned int') - tdSql.error(f'alter stable {stbname} modify column c9 double') - tdSql.error(f'alter stable {stbname} modify column c10 float') - tdSql.error(f'alter stable {stbname} modify column c11 int') - tdSql.error(f'alter stable {stbname} drop tag t0') - tdSql.execute(f'drop database {dbname}') - def alter_stable_tag_check(self,dbname,stbname,tbname): - tdSql.execute(f'create database if not exists {dbname}') - tdSql.execute(f'use {dbname}') - tdSql.execute( - f'create stable {stbname} (ts timestamp, c1 int) tags(ts_tag timestamp, t1 tinyint, t2 smallint, t3 int, \ - t4 bigint, t5 tinyint unsigned, t6 smallint unsigned, t7 int unsigned, t8 bigint unsigned, t9 float, t10 double, t11 bool,t12 binary(20),t13 nchar(20)) ') - tdSql.execute(f'create table {tbname} using {stbname} tags(now,1,2,3,4,5,6,7,8,9.9,10.1,true,"abcd","涛思数据")') - tdSql.execute(f'insert into {tbname} values(now,1)') - - tdSql.execute(f'alter stable {stbname} add tag t14 int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.execute(f'alter stable {stbname} add tag `t15` int') - tdSql.query(f'select t14 from {stbname}') - tdSql.checkRows(1) - tdSql.query(f'describe {stbname}') - tdSql.checkRows(18) - tdSql.execute(f'alter stable {stbname} drop tag t14') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(17) - tdSql.execute(f'alter stable {stbname} drop tag `t15`') - tdSql.query(f'describe {stbname}') - tdSql.checkRows(16) - tdSql.execute(f'alter stable {stbname} modify tag t12 binary(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,30) - tdSql.execute(f'alter stable {stbname} modify tag `t12` binary(35)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(14,2,35) - tdSql.error(f'alter stable {stbname} modify tag `t12` binary(34)') - tdSql.execute(f'alter stable {stbname} modify tag t13 nchar(30)') - tdSql.query(f'describe {stbname}') - tdSql.checkData(15,2,30) - tdSql.error(f'alter stable {stbname} modify tag t13 nchar(29)') - tdSql.execute(f'alter table {stbname} rename tag t1 t21') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t21') - tdSql.execute(f'alter table {stbname} rename tag `t21` t1') - tdSql.query(f'describe {stbname}') - tdSql.checkData(3,0,'t1') - - for i in ['bigint','unsigned int','float','double','binary(10)','nchar(10)']: - for j in [1,2,3]: - tdSql.error(f'alter stable {stbname} modify tag t{j} {i}') - for i in ['int','unsigned int','float','binary(10)','nchar(10)']: - tdSql.error(f'alter stable {stbname} modify tag t8 {i}') - tdSql.error(f'alter stable {stbname} modify tag t4 int') - tdSql.error(f'alter stable {stbname} drop column t0') - #!bug TD-16410 - # tdSql.error(f'alter stable {tbname} set tag t1=100 ') - # tdSql.execute(f'create table ntb (ts timestamp,c0 int)') - tdSql.error(f'alter stable ntb add column c2 ') - tdSql.execute(f'drop database {dbname}') def alter_stable_check(self): tdSql.prepare() tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for i in self.values_list: + tdSql.execute(f'insert into {self.ntbname} values({i})') for i in range(self.tbnum): tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_list[i]})') for j in self.values_list: @@ -211,6 +104,11 @@ class TDTestCase: tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)+1) tdSql.query(f'select {key} from {self.stbname}_{i}') tdSql.checkRows(len(self.values_list)) + for i in range(self.tbnum): + tdSql.error(f'alter stable {self.stbname}_{i} add column {key} {values}') + tdSql.error(f'alter stable {self.stbname}_{i} drop column {key}') + tdSql.error(f'alter stable {self.ntbname} add column {key} {values}') + tdSql.error(f'alter stable {self.ntbname} drop column {key}') tdSql.execute(f'alter stable {self.stbname} drop column {key}') tdSql.query(f'describe {self.stbname}') tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) @@ -231,6 +129,8 @@ class TDTestCase: tdSql.query(f'describe {self.stbname}_{i}') result = tdCom.getOneRow(1,'VARCHAR') tdSql.checkEqual(result[0][2],self.binary_length+1) + tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}') + tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') elif 'nchar' in values.lower(): v = f'nchar({self.binary_length+1})' v_error = f'nchar({self.binary_length-1})' @@ -243,17 +143,16 @@ class TDTestCase: tdSql.query(f'describe {self.stbname}_{i}') result = tdCom.getOneRow(1,'NCHAR') tdSql.checkEqual(result[0][2],self.binary_length+1) + tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}') + tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') else: for v in self.column_dict.values(): tdSql.error(f'alter stable {self.stbname} modify column {key} {v}') - pass + tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') + for i in range(self.tbnum): + tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}') def run(self): - # dbname = self.get_long_name(length=10, mode="letters") - # stbname = self.get_long_name(length=5, mode="letters") - # tbname = self.get_long_name(length=5, mode="letters") - # self.alter_stable_column_check(dbname,stbname,tbname) - # self.alter_stable_tag_check(dbname,stbname,tbname) self.alter_stable_check() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 6785fddc6f..4e1329c808 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -40,6 +40,7 @@ class TDTestCase: self.time_unit = ['b','u','a','s','m','h','d','w'] self.symbol = ['+','-','*','/'] self.error_values = [1.5,'abc','"abc"','!@','today()'] + self.db_percision = ['ms','us','ns'] def tbtype_check(self,tb_type): if tb_type == 'normal table' or tb_type == 'child table': tdSql.checkRows(len(self.values_list)) @@ -57,6 +58,7 @@ class TDTestCase: if v.lower() != 'timestamp': continue else: + sleep(1) tdSql.query(f'select * from {tbname} where {k}>=now()') tdSql.checkRows(0) tdSql.query(f'select * from {tbname} where {k} Date: Thu, 30 Jun 2022 16:50:57 +0800 Subject: [PATCH 03/27] update now.py and today.py --- tests/system-test/2-query/Now.py | 1 - tests/system-test/2-query/Today.py | 87 +++++++++++++++++++----------- 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 4e1329c808..3caf632209 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -58,7 +58,6 @@ class TDTestCase: if v.lower() != 'timestamp': continue else: - sleep(1) tdSql.query(f'select * from {tbname} where {k}>=now()') tdSql.checkRows(0) tdSql.query(f'select * from {tbname} where {k} self.today_date: - num_up += 1 - elif i[0] == self.today_date: - num_same += 1 - elif i[0] < self.today_date: - num_down += 1 + if precision == 'ms': + if int(i[0].timestamp())*1000 > int(self.today_ts)*1000: + num_up += 1 + elif int(i[0].timestamp())*1000 == int(self.today_ts)*1000: + num_same += 1 + elif int(i[0].timestamp())*1000 < int(self.today_ts)*1000: + num_down += 1 + elif precision == 'us': + if int(i[0].timestamp())*1000000 > int(self.today_ts)*1000000: + num_up += 1 + elif int(i[0].timestamp())*1000000 == int(self.today_ts)*1000000: + num_same += 1 + elif int(i[0].timestamp())*1000000 < int(self.today_ts)*1000000: + num_down += 1 + elif precision == 'ns': + if i[0] > int(self.today_ts)*1000000000: + num_up += 1 + elif i[0] == int(self.today_ts)*1000000000: + num_same += 1 + elif i[0] < int(self.today_ts)*1000000000: + num_down += 1 tdSql.query(f"select today() from {tbname}") tdSql.checkRows(len(values_list)*tb_num) tdSql.checkData(0, 0, str(self.today_date)) @@ -130,32 +151,36 @@ class TDTestCase: for i in range(num_same): tdSql.checkData(i, 0, str(self.today_date)) def today_check_ntb(self): - tdSql.prepare() - tdSql.execute(self.set_create_normaltable_sql(self.ntbname,self.column_dict)) - for i in self.values_list: - tdSql.execute( - f'insert into {self.ntbname} values({i})') - self.data_check(self.column_dict,self.ntbname,self.values_list) - tdSql.execute('drop database db') + for time_unit in self.db_percision: + print(time_unit) + tdSql.execute(f'create database db precision "{time_unit}"') + tdSql.execute('use db') + tdSql.execute(self.set_create_normaltable_sql(self.ntbname,self.column_dict)) + for i in self.values_list: + tdSql.execute( + f'insert into {self.ntbname} values({i})') + self.data_check(self.column_dict,self.ntbname,self.values_list,1,'tb',time_unit) + tdSql.execute('drop database db') def today_check_stb_tb(self): - tdSql.prepare() - tdSql.execute(self.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) - for i in range(self.tbnum): - tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})') - for j in self.values_list: - tdSql.execute(f'insert into {self.stbname}_{i} values ({j})') - # check child table - for i in range(self.tbnum): - self.data_check(self.column_dict,f'{self.stbname}_{i}',self.values_list) - # check stable - self.data_check(self.column_dict,self.stbname,self.values_list,self.tbnum,'stb') - tdSql.execute('drop database db') + for time_unit in self.db_percision: + print(time_unit) + tdSql.execute(f'create database db precision "{time_unit}"') + tdSql.execute('use db') + tdSql.execute(self.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f'create table if not exists {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})') + for j in self.values_list: + tdSql.execute(f'insert into {self.stbname}_{i} values ({j})') + # check child table + for i in range(self.tbnum): + self.data_check(self.column_dict,f'{self.stbname}_{i}',self.values_list,1,'tb',time_unit) + # check stable + self.data_check(self.column_dict,self.stbname,self.values_list,self.tbnum,'stb',time_unit) + tdSql.execute('drop database db') def run(self): # sourcery skip: extract-duplicate-method - tdLog.printNoPrefix("==========check today() for normal table ==========") self.today_check_ntb() - tdLog.printNoPrefix("==========check today() for stable and child table==========") self.today_check_stb_tb() def stop(self): From 91b5d707a0109dec5097ea62faee01bae26a1ab3 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Fri, 1 Jul 2022 09:53:01 +0800 Subject: [PATCH 04/27] update test case --- tests/system-test/2-query/Today.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py index 9a5f744782..e6199d629e 100644 --- a/tests/system-test/2-query/Today.py +++ b/tests/system-test/2-query/Today.py @@ -58,10 +58,6 @@ class TDTestCase: tag_sql += f"{k} {v}," create_stb_sql = f'create table {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})' return create_stb_sql - def datetime_transition(self,date,precision): - if precision == 'ns': - today_date = date*1000000000 - return today_date def data_check(self,column_dict={},tbname = '',values_list = [],tb_num = 1,tb = 'tb',precision = 'ms'): for k,v in column_dict.items(): From 0408224a8b45f40eb6686a06cd336505f18b4a8b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 13:05:59 +0800 Subject: [PATCH 05/27] test: remove case --- source/libs/index/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/index/CMakeLists.txt b/source/libs/index/CMakeLists.txt index 75eac2430f..33133d21ae 100644 --- a/source/libs/index/CMakeLists.txt +++ b/source/libs/index/CMakeLists.txt @@ -35,7 +35,7 @@ if (${BUILD_WITH_INVERTEDINDEX}) endif(${BUILD_WITH_INVERTEDINDEX}) -if (${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) +# if (${BUILD_TEST}) +# add_subdirectory(test) +# endif(${BUILD_TEST}) From 42a69c02df14b9606491b783c442152d3597a199 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 2 Jul 2022 15:33:06 +0800 Subject: [PATCH 06/27] update test case --- tests/system-test/1-insert/alter_table.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/alter_table.py b/tests/system-test/1-insert/alter_table.py index 0f7a830634..a2613c39e7 100644 --- a/tests/system-test/1-insert/alter_table.py +++ b/tests/system-test/1-insert/alter_table.py @@ -126,6 +126,7 @@ class TDTestCase: tdSql.execute(f'alter table {self.ntbname} rename column {key} {rename_str}') tdSql.query(f'select {rename_str} from {self.ntbname}') tdSql.checkRows(1) + tdSql.error(f'select {key} from {self.ntbname}') def alter_check_tb(self): tag_tinyint = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) @@ -277,7 +278,11 @@ class TDTestCase: else: for v in self.column_dict.values(): tdSql.error(f'alter table {self.stbname} modify column {key} {v}') - + for key,values in self.column_dict.items(): + rename_str = f'{tdCom.getLongName(constant.COL_NAME_LENGTH_MAX,"letters")}' + tdSql.error(f'alter table {self.stbname} rename column {key} {rename_str}') + for i in range(self.tbnum): + tdSql.error(f'alter table {self.stbname}_{i} rename column {key} {rename_str}') def run(self): self.alter_check_ntb() self.alter_check_tb() From afca287f99a60521255d996be32e1b2477f278f3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 16:59:34 +0800 Subject: [PATCH 07/27] refactor: temporarily comment out some code and use cases --- source/dnode/mnode/impl/src/mndDb.c | 23 +++++++++++ source/dnode/mnode/impl/src/mndDnode.c | 5 +++ source/dnode/mnode/impl/src/mndSnode.c | 14 +++++-- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++++ source/dnode/mnode/impl/test/db/db.cpp | 2 +- tests/script/jenkins/basic.txt | 52 ++++++++++++------------- 6 files changed, 74 insertions(+), 30 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c7db4b8e1c..4f958de121 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -549,18 +549,33 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) { +#if 1 + terrno = TSDB_CODE_OPS_NOT_SUPPORT; + return terrno; +#else pDb->cfg.buffer = pAlter->buffer; terrno = 0; +#endif } if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) { +#if 1 + terrno = TSDB_CODE_OPS_NOT_SUPPORT; + return terrno; +#else pDb->cfg.pages = pAlter->pages; terrno = 0; +#endif } if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) { +#if 1 + terrno = TSDB_CODE_OPS_NOT_SUPPORT; + return terrno; +#else pDb->cfg.pageSize = pAlter->pageSize; terrno = 0; +#endif } if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) { @@ -594,8 +609,12 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { } if (pAlter->strict >= 0 && pAlter->strict != pDb->cfg.strict) { +#if 1 + terrno = TSDB_CODE_OPS_NOT_SUPPORT; +#else pDb->cfg.strict = pAlter->strict; terrno = 0; +#endif } if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { @@ -604,9 +623,13 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { } if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) { +#if 1 + terrno = TSDB_CODE_OPS_NOT_SUPPORT; +#else pDb->cfg.replications = pAlter->replications; pDb->vgVersion++; terrno = 0; +#endif } return terrno; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5e78d0f434..8198445753 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -758,6 +758,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } } + if (numOfVnodes > 0) { + terrno = TSDB_CODE_OPS_NOT_SUPPORT; + goto _OVER; + } + code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 2dd8592bf8..d18a233d29 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -273,6 +273,9 @@ _OVER: } static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { +#if 1 + return TSDB_CODE_OPS_NOT_SUPPORT; +#else SMnode *pMnode = pReq->info.node; int32_t code = -1; SSnodeObj *pObj = NULL; @@ -315,6 +318,7 @@ _OVER: mndReleaseSnode(pMnode, pObj); mndReleaseDnode(pMnode, pDnode); return code; +#endif } static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) { @@ -386,9 +390,12 @@ _OVER: } static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SSnodeObj *pObj = NULL; +#if 1 + return TSDB_CODE_OPS_NOT_SUPPORT; +#else + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SSnodeObj *pObj = NULL; SMDropSnodeReq dropReq = {0}; if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { @@ -422,6 +429,7 @@ _OVER: mndReleaseSnode(pMnode, pObj); return code; +#endif } static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 5ca945624d..c53e831e84 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1219,6 +1219,9 @@ _OVER: } static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { +#if 1 + return TSDB_CODE_OPS_NOT_SUPPORT; +#else SMnode *pMnode = pReq->info.node; SDnodeObj *pNew1 = NULL; SDnodeObj *pNew2 = NULL; @@ -1412,6 +1415,7 @@ _OVER: mndReleaseDb(pMnode, pDb); return code; +#endif } int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) { @@ -1711,6 +1715,9 @@ _OVER: } static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { +#if 1 + return TSDB_CODE_OPS_NOT_SUPPORT; +#else SMnode *pMnode = pReq->info.node; int32_t code = -1; SArray *pArray = NULL; @@ -1759,6 +1766,7 @@ _OVER: taosArrayDestroy(pArray); return code; +#endif } bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index a1bab5d1d4..a3d129c7c4 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -93,7 +93,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, 0); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", ""); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9738fad24d..7499d9ac2e 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -9,8 +9,8 @@ ## ---- db ./test.sh -f tsim/db/alter_option.sim -./test.sh -f tsim/db/alter_replica_13.sim -./test.sh -f tsim/db/alter_replica_31.sim +# ./test.sh -f tsim/db/alter_replica_13.sim +# ./test.sh -f tsim/db/alter_replica_31.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim @@ -24,26 +24,26 @@ ./test.sh -f tsim/db/taosdlog.sim # ---- dnode -./test.sh -f tsim/dnode/balance_replica1.sim -./test.sh -f tsim/dnode/balance_replica3.sim -./test.sh -f tsim/dnode/balance1.sim -./test.sh -f tsim/dnode/balance2.sim -./test.sh -f tsim/dnode/balance3.sim -./test.sh -f tsim/dnode/balancex.sim +# ./test.sh -f tsim/dnode/balance_replica1.sim +# ./test.sh -f tsim/dnode/balance_replica3.sim +# ./test.sh -f tsim/dnode/balance1.sim +# ./test.sh -f tsim/dnode/balance2.sim +# ./test.sh -f tsim/dnode/balance3.sim +# ./test.sh -f tsim/dnode/balancex.sim ./test.sh -f tsim/dnode/create_dnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim -./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim -./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim -./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim -./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim -./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +# ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim +# ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim +# ./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +# ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +# ./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim ./test.sh -f tsim/dnode/offline_reason.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim -./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim -./test.sh -f tsim/dnode/vnode_clean.sim +# ./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +# ./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim +# ./test.sh -f tsim/dnode/vnode_clean.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim @@ -169,13 +169,13 @@ ./test.sh -f tsim/valgrind/checkError.sim -v # --- vnode -#./test.sh -f tsim/vnode/replica3_basic.sim -#./test.sh -f tsim/vnode/replica3_repeat.sim -./test.sh -f tsim/vnode/replica3_vgroup.sim -#./test.sh -f tsim/vnode/replica3_many.sim -#./test.sh -f tsim/vnode/replica3_import.sim -./test.sh -f tsim/vnode/stable_balance_replica1.sim -./test.sh -f tsim/vnode/stable_dnode2_stop.sim +# ./test.sh -f tsim/vnode/replica3_basic.sim +# ./test.sh -f tsim/vnode/replica3_repeat.sim +# ./test.sh -f tsim/vnode/replica3_vgroup.sim +# ./test.sh -f tsim/vnode/replica3_many.sim +# ./test.sh -f tsim/vnode/replica3_import.sim +# ./test.sh -f tsim/vnode/stable_balance_replica1.sim +# ./test.sh -f tsim/vnode/stable_dnode2_stop.sim ./test.sh -f tsim/vnode/stable_dnode2.sim ./test.sh -f tsim/vnode/stable_dnode3.sim ./test.sh -f tsim/vnode/stable_replica3_dnode6.sim From a535d374c662b43935811d2d4534ee5638959207 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 17:02:09 +0800 Subject: [PATCH 08/27] refactor: temporarily comment out some code and use cases --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 7499d9ac2e..15f117f7ed 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -71,7 +71,7 @@ ./test.sh -f tsim/qnode/basic1.sim # ---- snode -./test.sh -f tsim/snode/basic1.sim +# ./test.sh -f tsim/snode/basic1.sim # ---- bnode ./test.sh -f tsim/bnode/basic1.sim From 471fd29a0c246c7f284274a4f33288ed96356e12 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 17:48:42 +0800 Subject: [PATCH 09/27] test: remove case --- tests/script/tsim/sma/drop_sma.sim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index 73ada2de56..f6eaac2f7d 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -50,6 +50,13 @@ sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) t print --> create sma sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); +print --> show indexes +sql show d1.indexes +if $rows != 1 then + return -1 +endi + +return print --> drop stb sql drop table stb; From 932fb59204f0ae39db8dce2031c57283acda9be6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 17:49:43 +0800 Subject: [PATCH 10/27] test: remove case --- tests/script/tsim/sma/drop_sma.sim | 7 ------- tests/system-test/fulltest.sh | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index f6eaac2f7d..73ada2de56 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -50,13 +50,6 @@ sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) t print --> create sma sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); -print --> show indexes -sql show d1.indexes -if $rows != 1 then - return -1 -endi - -return print --> drop stb sql drop table stb; diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 9d02e305dc..bfd02ef6dd 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -116,7 +116,7 @@ python3 ./test.py -f 2-query/irate.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py -python3 ./test.py -f 6-cluster/5dnode1mnode.py +# python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 From 5f46a5a6b0b6c877600afa47bc0d446b18adec36 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 17:50:49 +0800 Subject: [PATCH 11/27] test: remove case --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 15f117f7ed..2b33067381 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -104,7 +104,7 @@ # ./test.sh -f tsim/stream/triggerSession0.sim ./test.sh -f tsim/stream/partitionby.sim ./test.sh -f tsim/stream/partitionby1.sim -./test.sh -f tsim/stream/schedSnode.sim +# ./test.sh -f tsim/stream/schedSnode.sim ./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/stream/ignoreExpiredData.sim From f2e1a3149c6379a19a3357756be6d4d7fe5818a2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 18:02:58 +0800 Subject: [PATCH 12/27] fxi: show indexes --- source/dnode/mnode/impl/src/mndShow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6014adbe95..2a8cbc4425 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -71,7 +71,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) { type = TSDB_MGMT_TABLE_FUNC; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) { - // type = TSDB_MGMT_TABLE_INDEX; + type = TSDB_MGMT_TABLE_INDEX; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) { type = TSDB_MGMT_TABLE_STB; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { From d42895d58158e3d5208e54dd9baa9e32ae31caba Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Jul 2022 18:34:00 +0800 Subject: [PATCH 13/27] fix: a problem of show indexes --- source/dnode/mnode/impl/src/mndShow.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 6014adbe95..2a8cbc4425 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -71,7 +71,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) { type = TSDB_MGMT_TABLE_FUNC; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) { - // type = TSDB_MGMT_TABLE_INDEX; + type = TSDB_MGMT_TABLE_INDEX; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) { type = TSDB_MGMT_TABLE_STB; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { From a6221e654a75096d69a9dc6d23438edc772a09d8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 18:45:07 +0800 Subject: [PATCH 14/27] fix: show indexes --- source/common/src/systable.c | 9 +++------ source/dnode/mnode/impl/src/mndSma.c | 25 +++++++++++-------------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 455f204542..79b556fd12 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -108,13 +108,10 @@ static const SSysDbTableSchema userFuncSchema[] = { }; static const SSysDbTableSchema userIdxSchema[] = { - {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; static const SSysDbTableSchema userStbsSchema[] = { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 6aea1f41d1..8e2391ba6e 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1131,14 +1131,14 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SSmaObj *pSma = NULL; int32_t cols = 0; - SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); - if (pDb == NULL) return 0; + SStbObj *pStb = mndAcquireStb(pMnode, pShow->db); + if (pStb == NULL) return 0; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); if (pShow->pIter == NULL) break; - if (pSma->dbUid != pDb->uid) { + if (pSma->stbUid != pStb->uid) { sdbRelease(pSdb, pSma); continue; } @@ -1147,22 +1147,16 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SName smaName = {0}; tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - - char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(n, (char *)tNameGetTableName(&smaName)); - cols++; - + char n0[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n0, (char *)tNameGetTableName(&smaName)); + SName stbName = {0}; tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName)); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false); + colDataAppend(pColInfo, numOfRows, (const char *)n0, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)n1, false); @@ -1170,11 +1164,14 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false); + numOfRows++; sdbRelease(pSdb, pSma); } - mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); pShow->numOfRows += numOfRows; return numOfRows; } From 687260701d2241ddd6a1cdd8f07e39edf5b0e802 Mon Sep 17 00:00:00 2001 From: Jason-Jia <714897623@qq.com> Date: Sat, 2 Jul 2022 18:53:16 +0800 Subject: [PATCH 15/27] Update alter_stable.py --- tests/system-test/1-insert/alter_stable.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/system-test/1-insert/alter_stable.py b/tests/system-test/1-insert/alter_stable.py index 4ff9d2a095..a4cec78138 100644 --- a/tests/system-test/1-insert/alter_stable.py +++ b/tests/system-test/1-insert/alter_stable.py @@ -107,8 +107,9 @@ class TDTestCase: for i in range(self.tbnum): tdSql.error(f'alter stable {self.stbname}_{i} add column {key} {values}') tdSql.error(f'alter stable {self.stbname}_{i} drop column {key}') - tdSql.error(f'alter stable {self.ntbname} add column {key} {values}') - tdSql.error(f'alter stable {self.ntbname} drop column {key}') + #! bug TD-16921 + #tdSql.error(f'alter stable {self.ntbname} add column {key} {values}') + #tdSql.error(f'alter stable {self.ntbname} drop column {key}') tdSql.execute(f'alter stable {self.stbname} drop column {key}') tdSql.query(f'describe {self.stbname}') tdSql.checkRows(len(self.column_dict)+len(self.tag_dict)) @@ -130,7 +131,8 @@ class TDTestCase: result = tdCom.getOneRow(1,'VARCHAR') tdSql.checkEqual(result[0][2],self.binary_length+1) tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}') - tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') + #! bug TD-16921 + # tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') elif 'nchar' in values.lower(): v = f'nchar({self.binary_length+1})' v_error = f'nchar({self.binary_length-1})' @@ -144,11 +146,12 @@ class TDTestCase: result = tdCom.getOneRow(1,'NCHAR') tdSql.checkEqual(result[0][2],self.binary_length+1) tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}') - tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') + #! bug TD-16921 + #tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') else: for v in self.column_dict.values(): tdSql.error(f'alter stable {self.stbname} modify column {key} {v}') - tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') + # tdSql.error(f'alter stable {self.ntbname} modify column {key} {v}') for i in range(self.tbnum): tdSql.error(f'alter stable {self.stbname}_{i} modify column {key} {v}') def run(self): @@ -159,4 +162,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From c08b3c93134f063ff31354bbbebeb0fe07ebd1cd Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Jul 2022 19:01:16 +0800 Subject: [PATCH 16/27] fix: a problem of show indexes --- source/libs/executor/src/scanoperator.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 89a47f4e2c..ee89bc1cb1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1308,6 +1308,13 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->scanCols); } +static int32_t getSysTableDbNameColId(const char* pTable) { + if (0 == strcmp(TSDB_INS_TABLE_USER_INDEXES, pTable)) { + return 1; + } + return TSDB_INS_USER_STABLES_DBNAME_COLID; +} + EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { int32_t code = TSDB_CODE_SUCCESS; ENodeType nType = nodeType(pNode); @@ -1329,7 +1336,7 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { } SColumnNode* node = (SColumnNode*)pNode; - if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) { + if (getSysTableDbNameColId(node->tableName) == node->colId) { *(int32_t*)pContext = 2; return DEAL_RES_CONTINUE; } From dd564e551caf754dadb149a2c5074f796a013620 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 2 Jul 2022 19:16:03 +0800 Subject: [PATCH 17/27] fix: a problem of show indexes --- source/common/src/systable.c | 28 +++++++++++------------ source/dnode/mnode/impl/src/mndSma.c | 30 ++++++++++++++++--------- source/libs/executor/src/scanoperator.c | 6 ++--- tools/taos-tools | 2 +- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 79b556fd12..b8844390d2 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -109,6 +109,7 @@ static const SSysDbTableSchema userFuncSchema[] = { static const SSysDbTableSchema userIdxSchema[] = { {.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, @@ -310,7 +311,7 @@ static const SSysDbTableSchema querySchema[] = { {.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, - {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, @@ -326,17 +327,17 @@ static const SSysDbTableSchema appSchema[] = { {.name = "app_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "ip", .bytes = TSDB_IPv4ADDR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "start_time", .bytes = 8 , .type = TSDB_DATA_TYPE_TIMESTAMP}, - {.name = "insert_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "insert_row", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "insert_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "insert_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "fetch_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "query_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "show_query", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "total_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "current_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "insert_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "insert_row", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "insert_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "insert_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "fetch_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "query_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "show_query", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "total_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "current_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, {.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; @@ -350,8 +351,7 @@ static const SSysTableMeta perfsMeta[] = { {TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)}, {TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)}, {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)}, - {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)} -}; + {TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}}; void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) { *pInfosTableMeta = infosMeta; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 8e2391ba6e..d151d23a89 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1131,14 +1131,14 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SSmaObj *pSma = NULL; int32_t cols = 0; - SStbObj *pStb = mndAcquireStb(pMnode, pShow->db); - if (pStb == NULL) return 0; + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); if (pShow->pIter == NULL) break; - if (pSma->stbUid != pStb->uid) { + if (pSma->dbUid != pDb->uid) { sdbRelease(pSdb, pSma); continue; } @@ -1147,19 +1147,27 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SName smaName = {0}; tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - char n0[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(n0, (char *)tNameGetTableName(&smaName)); - + char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName)); + + SName dbName = {0}; + tNameFromString(&dbName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + char n2[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n2, (char *)tNameGetTableName(&dbName)); + SName stbName = {0}; tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName)); + char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n0, false); + colDataAppend(pColInfo, numOfRows, (const char *)n1, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)n1, false); + colDataAppend(pColInfo, numOfRows, (const char *)n2, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)n3, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false); @@ -1171,7 +1179,7 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc sdbRelease(pSdb, pSma); } - mndReleaseStb(pMnode, pStb); + mndReleaseDb(pMnode, pDb); pShow->numOfRows += numOfRows; return numOfRows; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7cecc2a6ef..0a98ca9f66 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1309,9 +1309,9 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { } static int32_t getSysTableDbNameColId(const char* pTable) { - if (0 == strcmp(TSDB_INS_TABLE_USER_INDEXES, pTable)) { - return 1; - } + // if (0 == strcmp(TSDB_INS_TABLE_USER_INDEXES, pTable)) { + // return 1; + // } return TSDB_INS_USER_STABLES_DBNAME_COLID; } diff --git a/tools/taos-tools b/tools/taos-tools index 5fdd694621..7105027650 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 5fdd694621fbb7bd2d6102ff4feaec92a7001038 +Subproject commit 7105027650b51e701cfa1dac11b8fb42d447dd01 From 0a1299c8d078b42d0f530610ac35656327b019f4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 19:23:07 +0800 Subject: [PATCH 18/27] fix: show indexes --- source/dnode/mnode/impl/src/mndSma.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index d151d23a89..da5b8cb48e 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -1150,10 +1150,8 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName)); - SName dbName = {0}; - tNameFromString(&dbName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - char n2[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(n2, (char *)tNameGetTableName(&dbName)); + char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n2, (char *)mndGetDbStr(pDb->name)); SName stbName = {0}; tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); From fbcf0dab2c8f621fdeb684ae4c18248c529c1e7e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 19:24:06 +0800 Subject: [PATCH 19/27] test: show indexes --- tests/script/tsim/sma/drop_sma.sim | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index 73ada2de56..67e39023b5 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -53,6 +53,22 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in print --> drop stb sql drop table stb; +print --> show sma +sql show indexes from d1 +if $rows != 1 then + return -1 +endi +if $data[0][0] != sma_index_name1 then + return -1 +endi +if $data[0][0] != d1 then + return -1 +endi +if $data[0][0] != stb then + return -1 +endi + + print ========== step4 repeat print --> create stb @@ -64,6 +80,21 @@ sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) in print --> drop stb sql drop table stb; +print --> show sma +sql show indexes from stb +if $rows != 1 then + return -1 +endi +if $data[0][0] != sma_index_name1 then + return -1 +endi +if $data[0][0] != d1 then + return -1 +endi +if $data[0][0] != stb then + return -1 +endi + print ========== step5 sql drop database if exists db; sql create database db duration 300; From 900216f77f16f2447d0d3681c976aa8a4cfa7293 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 19:42:05 +0800 Subject: [PATCH 20/27] test: show indexes --- tests/script/tsim/sma/drop_sma.sim | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/script/tsim/sma/drop_sma.sim b/tests/script/tsim/sma/drop_sma.sim index 67e39023b5..78f86f6e19 100644 --- a/tests/script/tsim/sma/drop_sma.sim +++ b/tests/script/tsim/sma/drop_sma.sim @@ -50,24 +50,23 @@ sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) t print --> create sma sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); -print --> drop stb -sql drop table stb; - print --> show sma -sql show indexes from d1 +sql show indexes from stb from d1; if $rows != 1 then return -1 endi if $data[0][0] != sma_index_name1 then return -1 endi -if $data[0][0] != d1 then +if $data[0][1] != d1 then return -1 endi -if $data[0][0] != stb then +if $data[0][2] != stb then return -1 endi +print --> drop stb +sql drop table stb; print ========== step4 repeat @@ -77,24 +76,24 @@ sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) t print --> create sma sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(6m,10s) sliding(6m); -print --> drop stb -sql drop table stb; - print --> show sma -sql show indexes from stb +sql show indexes from stb from d1; if $rows != 1 then return -1 endi if $data[0][0] != sma_index_name1 then return -1 endi -if $data[0][0] != d1 then +if $data[0][1] != d1 then return -1 endi -if $data[0][0] != stb then +if $data[0][2] != stb then return -1 endi +print --> drop stb +sql drop table stb; + print ========== step5 sql drop database if exists db; sql create database db duration 300; From bf61e7490868da7f9512d5e304d8191e70a0453a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 2 Jul 2022 20:10:22 +0800 Subject: [PATCH 21/27] test: add test case for tmq --- .../7-tmq/tmqConsFromTsdb-mutilVg.py | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py new file mode 100644 index 0000000000..a4d6648276 --- /dev/null +++ b/tests/system-test/7-tmq/tmqConsFromTsdb-mutilVg.py @@ -0,0 +1,242 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("create stb") + tmqCom.create_stable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"]) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict['ctbStartIdx']) + tdLog.info("insert data") + tmqCom.insert_data_interlaceByMultiTbl(tsql=tdSql,dbName=paraDict["dbName"],ctbPrefix=paraDict["ctbPrefix"], + ctbNum=paraDict["ctbNum"],rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"], + startTs=paraDict["startTs"],ctbStartIdx=paraDict['ctbStartIdx']) + + tdLog.info("restart taosd to ensure that the data falls into the disk") + tdDnodes.stop(1) + # tdDnodes.start(1) + tdDnodes.starttaosd(1) + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + # tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 3, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 1} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + topicNameList = ['topic1'] + expectRowsList = [] + tmqCom.initConsumerTable() + + tdLog.info("create topics from stb with filter") + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + # sqlString = "create topic %s as stable %s" %(topicNameList[0], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + totalRowsInserted = expectRowsList[0] + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 1 + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] / 3) + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:1000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 0") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if not (expectrowcnt <= resultList[0] and totalRowsInserted >= resultList[0]): + tdLog.info("act consume rows: %d, expect consume rows between %d and %d"%(resultList[0], expectrowcnt, totalRowsInserted)) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + firstConsumeRows = resultList[0] + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + consumerId = 2 + expectrowcnt = math.ceil(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2/3) + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor 1") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + tdLog.info("wait the consume result") + + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + actConsumeTotalRows = firstConsumeRows + resultList[0] + + if not (expectrowcnt >= resultList[0] and totalRowsInserted == actConsumeTotalRows): + tdLog.info("act consume rows, first: %d, second: %d "%(firstConsumeRows, resultList[0])) + tdLog.info("and sum of two consume rows: %d should be equal to total inserted rows: %d"%(actConsumeTotalRows, totalRowsInserted)) + tdLog.exit("%d tmq consume rows error!"%consumerId) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + tdSql.prepare() + self.prepareTestEnv() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 348c530d5876b28e2995790653677f5d14957baf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 20:10:32 +0800 Subject: [PATCH 22/27] test: remove some case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ead3970b48..7629bcf173 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -117,7 +117,7 @@ python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py -python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 +#python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 From 31bbef82d16f325469051e8e6f38415d59277727 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 2 Jul 2022 20:18:40 +0800 Subject: [PATCH 23/27] fix(tmq): seek snapshot offset --- source/dnode/mnode/impl/src/mndStream.c | 1 - source/dnode/vnode/src/tq/tq.c | 187 ------------------------ source/libs/executor/src/executorimpl.c | 33 ++++- tests/system-test/fulltest.sh | 1 + 4 files changed, 26 insertions(+), 196 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 08e108803a..c2125f75f8 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -247,7 +247,6 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name)); pObj->status = 0; - // TODO pObj->igExpired = pCreate->igExpired; pObj->trigger = pCreate->triggerType; pObj->triggerParam = pCreate->maxDelay; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7c4a7d520d..5ce3cfab45 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -406,193 +406,6 @@ OVER: return code; } -#if 0 -int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { - SMqPollReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int64_t timeout = pReq->timeout; - int32_t reqEpoch = pReq->epoch; - int64_t fetchOffset; - int32_t code = 0; - - // get offset to fetch message - if (pReq->currentOffset >= 0) { - fetchOffset = pReq->currentOffset + 1; - } else { - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); - if (pOffset != NULL) { - ASSERT(pOffset->val.type == TMQ_OFFSET__LOG); - tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey, - TD_VID(pTq->pVnode), pOffset->val.version); - fetchOffset = pOffset->val.version + 1; - } else { - if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { - fetchOffset = walGetFirstVer(pTq->pWal); - } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) { - fetchOffset = walGetCommittedVer(pTq->pWal); - } else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__NONE) { - tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode), - pReq->subKey); - terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; - return -1; - } - tqDebug("consumer %ld, restore offset of %s on vg %d failed, config is %ld, set to %ld", consumerId, pReq->subKey, - TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); - } - } - - tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %ld fetch offset %ld", consumerId, - pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset); - - STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); - /*ASSERT(pHandle);*/ - if (pHandle == NULL) { - tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode), - pReq->subKey); - return -1; - } - - if (pHandle->consumerId != consumerId) { - tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld", - consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); - return -1; - } - - int32_t consumerEpoch = atomic_load_32(&pHandle->epoch); - while (consumerEpoch < reqEpoch) { - consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch); - } - - SMqDataBlkRsp rsp = {0}; - rsp.reqOffset = pReq->currentOffset; - - rsp.blockData = taosArrayInit(0, sizeof(void*)); - rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); - - if (rsp.blockData == NULL || rsp.blockDataLen == NULL) { - return -1; - } - - rsp.withTbName = pReq->withTbName; - if (rsp.withTbName) { - rsp.blockTbName = taosArrayInit(0, sizeof(void*)); - } - - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - rsp.withSchema = false; - } else { - rsp.withSchema = true; - rsp.blockSchema = taosArrayInit(0, sizeof(void*)); - } - -#if 1 - if (pReq->useSnapshot) { - // TODO set ver into snapshot - int64_t lastVer = walGetCommittedVer(pTq->pWal); - if (rsp.reqOffset < lastVer) { - tqInfo("retrieve using snapshot req offset %ld last ver %ld", rsp.reqOffset, lastVer); - tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId); - - if (rsp.blockNum != 0) { - rsp.withTbName = false; - rsp.rspOffset = lastVer; - tqInfo("direct send by snapshot req offset %ld rsp offset %ld", rsp.reqOffset, rsp.rspOffset); - fetchOffset = lastVer; - goto SEND_RSP; - } - } - } -#endif - - SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); - if (pHeadWithCkSum == NULL) { - return -1; - } - - walSetReaderCapacity(pHandle->pWalReader, 2048); - - while (1) { - consumerEpoch = atomic_load_32(&pHandle->epoch); - if (consumerEpoch > reqEpoch) { - tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d", - consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch); - break; - } - - if (tqFetchLog(pTq, pHandle, &fetchOffset, &pHeadWithCkSum) < 0) { - // TODO add push mgr - break; - } - - SWalCont* pHead = &pHeadWithCkSum->head; - - tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); - - if (pHead->msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - - if (tqDataExec(pTq, &pHandle->execHandle, pCont, &rsp, workerId) < 0) { - /*ASSERT(0);*/ - } - } else { - ASSERT(pHandle->fetchMeta); - ASSERT(IS_META_MSG(pHead->msgType)); - tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); - SMqMetaRsp metaRsp = {0}; - metaRsp.reqOffset = pReq->currentOffset; - metaRsp.rspOffset = fetchOffset; - metaRsp.resMsgType = pHead->msgType; - metaRsp.metaRspLen = pHead->bodyLen; - metaRsp.metaRsp = pHead->body; - if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { - code = -1; - goto OVER; - } - code = 0; - goto OVER; - } - - // TODO batch optimization: - // TODO continue scan until meeting batch requirement - if (rsp.blockNum > 0 /* threshold */) { - break; - } else { - fetchOffset++; - } - } - - taosMemoryFree(pHeadWithCkSum); - -SEND_RSP: - ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); - ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); - if (rsp.withSchema) { - ASSERT(taosArrayGetSize(rsp.blockSchema) == rsp.blockNum); - } - - rsp.rspOffset = fetchOffset; - - if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) { - code = -1; - } -OVER: - // TODO wrap in destroy func - taosArrayDestroy(rsp.blockDataLen); - taosArrayDestroyP(rsp.blockData, (FDelete)taosMemoryFree); - - if (rsp.withSchema) { - taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); - } - - if (rsp.withTbName) { - taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree); - } - - return code; -} -#endif - int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8850ac1c7a..20e507b040 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1422,7 +1422,7 @@ void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t return; } #ifdef BUF_PAGE_DEBUG - qDebug("page_setbuf, groupId:%"PRIu64, groupId); + qDebug("page_setbuf, groupId:%" PRIu64, groupId); #endif doSetTableGroupOutputBuf(pOperator, pAggInfo, numOfOutput, groupId); @@ -1570,9 +1570,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; -// if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full -// break; -// } + // if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full + // break; + // } } qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, @@ -2868,7 +2868,24 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { pInfo->cond.twindows[0].skey = oldSkey; pInfo->scanTimes = 0; pInfo->curTWinIdx = 0; + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); + bool found = false; + for (int32_t i = 0; i < tableSz; i++) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); + if (pTableInfo->uid == uid) { + found = true; + pInfo->currentTable = i; + } + } + // TODO after processing drop, + ASSERT(found); + qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, + pInfo->currentTable, tableSz); } + return TSDB_CODE_SUCCESS; } else { @@ -4107,8 +4124,8 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } else { isNull[index++] = 0; char* data = nodesGetValueFromNode(pValue); - if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){ - if(tTagIsJson(data)){ + if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(data)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; taosMemoryFree(keyBuf); nodesClearList(groupNew); @@ -4173,7 +4190,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - if(code){ + if (code) { pTaskInfo->code = code; return NULL; } @@ -4202,7 +4219,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo }; if (pHandle) { int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - if(code){ + if (code) { pTaskInfo->code = code; return NULL; } diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ead3970b48..e2f7003edf 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -157,3 +157,4 @@ python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqAlterSchema.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py From cee0bf30f30063c506e54c814dfa5fb8833a74f1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 20:23:59 +0800 Subject: [PATCH 24/27] test: remove some case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 7629bcf173..a7ab328b55 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -116,7 +116,7 @@ python3 ./test.py -f 2-query/irate.py python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py -python3 ./test.py -f 6-cluster/5dnode1mnode.py +#python3 ./test.py -f 6-cluster/5dnode1mnode.py #python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 #python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 From 121da02c0392db1578d27ec50dc51faea5c94858 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 2 Jul 2022 20:24:19 +0800 Subject: [PATCH 25/27] refactor: adjust max vgroups per db --- include/util/tdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 5befa6a67f..7c7413a421 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -287,7 +287,7 @@ typedef enum ELogicConditionType { #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta #define TSDB_MIN_VNODES_PER_DB 1 -#define TSDB_MAX_VNODES_PER_DB 4096 +#define TSDB_MAX_VNODES_PER_DB 1024 #define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB #define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB From d493fbed985249fff7aa94d24226b951aa432809 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 3 Jul 2022 11:54:19 +0800 Subject: [PATCH 26/27] fix invalid read/write --- source/dnode/mgmt/node_mgmt/src/dmProc.c | 20 +++++++++---------- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/libs/transport/src/transSvr.c | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index 72878d0d85..cbf13924d7 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -87,8 +87,8 @@ static SProcQueue *dmInitProcQueue(SProc *proc, char *ptr, int32_t size) { static void dmCleanupProcQueue(SProcQueue *queue) {} static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg *pMsg, EProcFuncType ftype) { - const void *pHead = pMsg; - const void *pBody = pMsg->pCont; + const void * pHead = pMsg; + const void * pBody = pMsg->pCont; const int16_t rawHeadLen = sizeof(SRpcMsg); const int32_t rawBodyLen = pMsg->contLen; const int16_t headLen = CEIL8(rawHeadLen); @@ -257,7 +257,7 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper) { proc->wrapper = pWrapper; proc->name = pWrapper->name; - SShm *shm = &proc->shm; + SShm * shm = &proc->shm; int32_t cstart = 0; int32_t csize = CEIL8(shm->size / 2); int32_t pstart = csize; @@ -281,13 +281,13 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper) { } static void *dmConsumChildQueue(void *param) { - SProc *proc = param; + SProc * proc = param; SMgmtWrapper *pWrapper = proc->wrapper; - SProcQueue *queue = proc->cqueue; + SProcQueue * queue = proc->cqueue; int32_t numOfMsgs = 0; int32_t code = 0; EProcFuncType ftype = DND_FUNC_REQ; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; dDebug("node:%s, start to consume from cqueue", proc->name); do { @@ -324,13 +324,13 @@ static void *dmConsumChildQueue(void *param) { } static void *dmConsumParentQueue(void *param) { - SProc *proc = param; + SProc * proc = param; SMgmtWrapper *pWrapper = proc->wrapper; - SProcQueue *queue = proc->pqueue; + SProcQueue * queue = proc->pqueue; int32_t numOfMsgs = 0; int32_t code = 0; EProcFuncType ftype = DND_FUNC_REQ; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; dDebug("node:%s, start to consume from pqueue", proc->name); do { @@ -353,7 +353,7 @@ static void *dmConsumParentQueue(void *param) { rpcRegisterBrokenLinkArg(pMsg); } else if (ftype == DND_FUNC_RELEASE) { dmRemoveProcRpcHandle(proc, pMsg->info.handle); - rpcReleaseHandle(pMsg->info.handle, (int8_t)pMsg->code); + rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER); } else { dError("node:%s, invalid ftype:%d from pqueue", proc->name, ftype); rpcFreeCont(pMsg->pCont); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 35d478177a..df3c9c4e88 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -245,7 +245,7 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { SRpcMsg msg = {.code = type, .info = *pHandle}; dmPutToProcPQueue(&pWrapper->proc, &msg, DND_FUNC_RELEASE); } else { - rpcReleaseHandle(pHandle->handle, type); + rpcReleaseHandle(pHandle, type); } } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index d32156dd0d..39cda4c9a9 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1029,9 +1029,9 @@ void transUnrefSrvHandle(void* handle) { } void transReleaseSrvHandle(void* handle) { - SExHandle* exh = handle; - int64_t refId = exh->refId; - + SRpcHandleInfo* info = handle; + SExHandle* exh = info->handle; + int64_t refId = info->refId; ASYNC_CHECK_HANDLE(exh, refId); SWorkThrd* pThrd = exh->pThrd; From 87f28ab358705dc504da87081cf1a01cb0a29beb Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Sun, 3 Jul 2022 15:01:44 +0800 Subject: [PATCH 27/27] =?UTF-8?q?Revert=20"fix=EF=BC=9A=20avoid=20invalid?= =?UTF-8?q?=20read"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- source/dnode/mgmt/node_mgmt/src/dmProc.c | 20 +++++++++---------- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 2 +- source/libs/transport/src/transSvr.c | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmProc.c b/source/dnode/mgmt/node_mgmt/src/dmProc.c index cbf13924d7..72878d0d85 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmProc.c +++ b/source/dnode/mgmt/node_mgmt/src/dmProc.c @@ -87,8 +87,8 @@ static SProcQueue *dmInitProcQueue(SProc *proc, char *ptr, int32_t size) { static void dmCleanupProcQueue(SProcQueue *queue) {} static inline int32_t dmPushToProcQueue(SProc *proc, SProcQueue *queue, SRpcMsg *pMsg, EProcFuncType ftype) { - const void * pHead = pMsg; - const void * pBody = pMsg->pCont; + const void *pHead = pMsg; + const void *pBody = pMsg->pCont; const int16_t rawHeadLen = sizeof(SRpcMsg); const int32_t rawBodyLen = pMsg->contLen; const int16_t headLen = CEIL8(rawHeadLen); @@ -257,7 +257,7 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper) { proc->wrapper = pWrapper; proc->name = pWrapper->name; - SShm * shm = &proc->shm; + SShm *shm = &proc->shm; int32_t cstart = 0; int32_t csize = CEIL8(shm->size / 2); int32_t pstart = csize; @@ -281,13 +281,13 @@ int32_t dmInitProc(struct SMgmtWrapper *pWrapper) { } static void *dmConsumChildQueue(void *param) { - SProc * proc = param; + SProc *proc = param; SMgmtWrapper *pWrapper = proc->wrapper; - SProcQueue * queue = proc->cqueue; + SProcQueue *queue = proc->cqueue; int32_t numOfMsgs = 0; int32_t code = 0; EProcFuncType ftype = DND_FUNC_REQ; - SRpcMsg * pMsg = NULL; + SRpcMsg *pMsg = NULL; dDebug("node:%s, start to consume from cqueue", proc->name); do { @@ -324,13 +324,13 @@ static void *dmConsumChildQueue(void *param) { } static void *dmConsumParentQueue(void *param) { - SProc * proc = param; + SProc *proc = param; SMgmtWrapper *pWrapper = proc->wrapper; - SProcQueue * queue = proc->pqueue; + SProcQueue *queue = proc->pqueue; int32_t numOfMsgs = 0; int32_t code = 0; EProcFuncType ftype = DND_FUNC_REQ; - SRpcMsg * pMsg = NULL; + SRpcMsg *pMsg = NULL; dDebug("node:%s, start to consume from pqueue", proc->name); do { @@ -353,7 +353,7 @@ static void *dmConsumParentQueue(void *param) { rpcRegisterBrokenLinkArg(pMsg); } else if (ftype == DND_FUNC_RELEASE) { dmRemoveProcRpcHandle(proc, pMsg->info.handle); - rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER); + rpcReleaseHandle(pMsg->info.handle, (int8_t)pMsg->code); } else { dError("node:%s, invalid ftype:%d from pqueue", proc->name, ftype); rpcFreeCont(pMsg->pCont); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index df3c9c4e88..35d478177a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -245,7 +245,7 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { SRpcMsg msg = {.code = type, .info = *pHandle}; dmPutToProcPQueue(&pWrapper->proc, &msg, DND_FUNC_RELEASE); } else { - rpcReleaseHandle(pHandle, type); + rpcReleaseHandle(pHandle->handle, type); } } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 39cda4c9a9..d32156dd0d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1029,9 +1029,9 @@ void transUnrefSrvHandle(void* handle) { } void transReleaseSrvHandle(void* handle) { - SRpcHandleInfo* info = handle; - SExHandle* exh = info->handle; - int64_t refId = info->refId; + SExHandle* exh = handle; + int64_t refId = exh->refId; + ASYNC_CHECK_HANDLE(exh, refId); SWorkThrd* pThrd = exh->pThrd;