From bd643aaefd63e1073525ea3f4dbc5e7d283b9dff Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 20 Jul 2022 16:43:46 +0800 Subject: [PATCH 01/36] test:add vnode test case --- .../4dnode1mnode_basic_createDb_replica1.py | 138 ++++++++++++++ ...4dnode1mnode_basic_replica1_insertdatas.py | 179 ++++++++++++++++++ 2 files changed, 317 insertions(+) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py new file mode 100644 index 0000000000..28d5b47b4f --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py @@ -0,0 +1,138 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-3]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def getConnection(self, dnode): + host = dnode.cfgDict["fqdn"] + port = dnode.cfgDict["serverPort"] + config_dir = dnode.cfgDir + return taos.connect(host=host, port=int(port), config=config_dir) + + + def run(self): + self.check_setup_cluster_status() + self.create_db_check_vgroups() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py new file mode 100644 index 0000000000..e113e2a6e5 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py @@ -0,0 +1,179 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 1 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-3]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_db_replica_1_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(tb_nums): + sub_tbname = "sub_tb_{}".format(i) + tdSql.execute("create table {} using stb1 tags({})".format(sub_tbname,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + 1000*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== create database {} and insert rows execute end =====".format(dbname)) + + def check_insert_status(self, dbname, tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.query("select count(*) from {}.{}".format(dbname,'stb1')) + tdSql.checkData(0 , 0 , tb_nums*row_nums) + tdSql.query("select distinct tbname from {}.{}".format(dbname,'stb1')) + tdSql.checkRows(tb_nums) + + def run(self): + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.create_db_replica_1_insertdatas(self.db_name , self.replica , self.vgroups , self.tb_nums , self.row_nums) + self.check_insert_status(self.db_name , self.tb_nums , self.row_nums) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 14dfe117910668ed344a3dd42dbc2a00852bb02b Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Wed, 20 Jul 2022 20:00:54 +0800 Subject: [PATCH 02/36] add case about vote leader about replica 3 --- .../4dnode1mnode_basic_createDb_replica1.py | 2 +- ...4dnode1mnode_basic_replica1_insertdatas.py | 2 +- .../4dnode1mnode_basic_replica3_vgroups.py | 206 ++++++++++++++++++ 3 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py index 28d5b47b4f..e6192ba313 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py @@ -108,7 +108,7 @@ class TDTestCase: for vgroup_info in tdSql.queryResult: vgroup_id = vgroup_info[0] tmp_list = [] - for role in vgroup_info[3:-3]: + for role in vgroup_info[3:-4]: if role in ['leader','follower']: tmp_list.append(role) vgroups_infos[vgroup_id]=tmp_list diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py index e113e2a6e5..d5fef08945 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py @@ -114,7 +114,7 @@ class TDTestCase: for vgroup_info in tdSql.queryResult: vgroup_id = vgroup_info[0] tmp_list = [] - for role in vgroup_info[3:-3]: + for role in vgroup_info[3:-4]: if role in ['leader','follower']: tmp_list.append(role) vgroups_infos[vgroup_id]=tmp_list diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py new file mode 100644 index 0000000000..5529a5e256 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py @@ -0,0 +1,206 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 1 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + self.max_vote_time_cost = 10 # seconds + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def check_vgroups_init_done(self,dbname): + + status = True + + tdSql.query("show {}.vgroups".format(dbname)) + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + vgroup_status = [] + for ind , role in enumerate(vgroup_info[3:-4]): + + if ind%2==0: + continue + else: + vgroup_status.append(role) + if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=2: + status = False + return status + return status + + + def vote_leader_time_costs(self,dbname): + start = time.time() + status = self.check_vgroups_init_done(dbname) + while not status: + time.sleep(0.1) + status = self.check_vgroups_init_done(dbname) + + # tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname)) + end = time.time() + cost_time = end - start + tdLog.info(" ==== database %s vote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) + # os.system("taos -s 'show {}.vgroups;'".format(dbname)) + if cost_time >= self.max_vote_time_cost: + tdLog.exit(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + + + return cost_time + + def test_init_vgroups_time_costs(self): + + tdLog.info(" ====start check time cost about vgroups vote leaders ==== ") + tdLog.info(" ==== current max time cost is set value : {} =======".format(self.max_vote_time_cost)) + + # create database replica 3 vgroups 1 + + db1 = 'db_1' + create_db_replica_3_vgroups_1 = "create database {} replica 3 vgroups 1".format(db1) + tdLog.info('=======database {} replica 3 vgroups 1 ======'.format(db1)) + tdSql.execute(create_db_replica_3_vgroups_1) + self.vote_leader_time_costs(db1) + + # create database replica 3 vgroups 10 + db2 = 'db_2' + create_db_replica_3_vgroups_10 = "create database {} replica 3 vgroups 10".format(db2) + tdLog.info('=======database {} replica 3 vgroups 10 ======'.format(db2)) + tdSql.execute(create_db_replica_3_vgroups_10) + self.vote_leader_time_costs(db2) + + # create database replica 3 vgroups 100 + db3 = 'db_3' + create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 100".format(db3) + tdLog.info('=======database {} replica 3 vgroups 100 ======'.format(db3)) + tdSql.execute(create_db_replica_3_vgroups_100) + self.vote_leader_time_costs(db3) + + + + def run(self): + self.check_setup_cluster_status() + self.test_init_vgroups_time_costs() + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From ff06586b2c2c6406519c0049bd1c13408f1cfd60 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 21 Jul 2022 16:04:13 +0800 Subject: [PATCH 03/36] revote leader when stop one dnode --- ...de1mnode_basic_replica3_vgroups_stopOne.py | 364 ++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py new file mode 100644 index 0000000000..3244b6bd7b --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py @@ -0,0 +1,364 @@ +# author : wenzhouwww +from errno import ESOCKTNOSUPPORT +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import random +import socket +import subprocess + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 1 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + self.max_vote_time_cost = 10 # seconds + self.stop_dnode = None + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def _get_stop_dnode(self): + only_dnode_list = self.dnode_list.keys() - self.mnode_list.keys() + self.stop_dnode = random.sample(only_dnode_list , 1 )[0] + return self.stop_dnode + + + def check_vgroups_revote_leader(self,dbname): + + status = True + stop_dnode_id = self.dnode_list[self.stop_dnode][0] + + tdSql.query("show {}.vgroups".format(dbname)) + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + vgroup_status = [] + vgroups_leader_follower = vgroup_info[3:-4] + for ind , role in enumerate(vgroups_leader_follower): + + if ind%2==0: + if role == stop_dnode_id and vgroups_leader_follower[ind+1]=="offline": + tdLog.info("====== dnode {} has offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode)) + elif role == stop_dnode_id : + tdLog.exit("====== dnode {} has not offline , endpoint is {}".format(stop_dnode_id , self.stop_dnode)) + else: + continue + else: + vgroup_status.append(role) + if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=1 or vgroup_status.count("offline")!=1: + status = False + return status + return status + + + def wait_stop_dnode_OK(self): + + def _get_status(): + + status = "" + tdSql.query("show dnodes") + dnode_infos = tdSql.queryResult + for dnode_info in dnode_infos: + endpoint = dnode_info[1] + dnode_status = dnode_info[4] + if endpoint == self.stop_dnode: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , endpoint is {}".format(self.stop_dnode)) + + def wait_start_dnode_OK(self): + + def _get_status(): + + status = "" + tdSql.query("show dnodes") + dnode_infos = tdSql.queryResult + for dnode_info in dnode_infos: + endpoint = dnode_info[1] + dnode_status = dnode_info[4] + if endpoint == self.stop_dnode: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , endpoint is {}".format(self.stop_dnode)) + + + + def random_stop_One_dnode(self): + self.stop_dnode = self._get_stop_dnode() + stop_dnode_id = self.dnode_list[self.stop_dnode][0] + tdLog.info(" ==== dnode {} will offline ,endpoints is {} ====".format(stop_dnode_id , self.stop_dnode)) + tdDnodes=cluster.dnodes + tdDnodes[stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK() + # os.system("taos -s 'show dnodes;'") + + def Restart_stop_dnode(self): + + tdDnodes=cluster.dnodes + stop_dnode_id = self.dnode_list[self.stop_dnode][0] + tdDnodes[stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + # os.system("taos -s 'show dnodes;'") + + def check_vgroups_init_done(self,dbname): + + status = True + + tdSql.query("show {}.vgroups".format(dbname)) + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + vgroup_status = [] + for ind , role in enumerate(vgroup_info[3:-4]): + + if ind%2==0: + continue + else: + vgroup_status.append(role) + if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=2: + status = False + return status + return status + + def vote_leader_time_costs(self,dbname): + start = time.time() + status = self.check_vgroups_init_done(dbname) + while not status: + time.sleep(0.1) + status = self.check_vgroups_init_done(dbname) + + # tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname)) + end = time.time() + cost_time = end - start + tdLog.info(" ==== database %s vote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) + # os.system("taos -s 'show {}.vgroups;'".format(dbname)) + if cost_time >= self.max_vote_time_cost: + tdLog.exit(" ==== database %s vote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + + return cost_time + + + def revote_leader_time_costs(self,dbname): + start = time.time() + + status = self.check_vgroups_revote_leader(dbname) + while not status: + time.sleep(0.1) + status = self.check_vgroups_revote_leader(dbname) + + # tdLog.info("=== database {} show vgroups vote the leader is in progress ===".format(dbname)) + end = time.time() + cost_time = end - start + tdLog.info(" ==== database %s revote the leaders success , cost time is %.3f second ====="%(dbname,cost_time) ) + # os.system("taos -s 'show {}.vgroups;'".format(dbname)) + if cost_time >= self.max_vote_time_cost: + tdLog.exit(" ==== database %s revote the leaders cost too large time , cost time is %.3f second ===="%(dbname,cost_time) ) + + + return cost_time + + def exec_revote_action(self,dbname): + + tdSql.query("show {}.vgroups".format(dbname)) + before_revote = tdSql.queryResult + + before_vgroups = set() + for vgroup_info in before_revote: + before_vgroups.add(vgroup_info[3:-4]) + + self.random_stop_One_dnode() + tdSql.query("show {}.vgroups".format(dbname)) + after_revote = tdSql.queryResult + + after_vgroups = set() + for vgroup_info in after_revote: + after_vgroups.add(vgroup_info[3:-4]) + + vote_act = set(set(after_vgroups)-set(before_vgroups)) + if not vote_act: + tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + else: + for vgroup_info in vote_act: + for ind , role in enumerate(vgroup_info): + if role==self.dnode_list[self.stop_dnode][0]: + + if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: + tdLog.info(" === revote leader ok , leader is {} now ====".format(list(vgroup_info).index("leader")-1)) + elif vgroup_info[ind+1] !="offline": + tdLog.exit(" === dnode {} should be offline ".format(self.stop_dnode)) + else: + continue + break + + + + self.revote_leader_time_costs(dbname) + self.Restart_stop_dnode() + def test_init_vgroups_time_costs(self): + + tdLog.info(" ====start check time cost about vgroups vote leaders ==== ") + tdLog.info(" ==== current max time cost is set value : {} =======".format(self.max_vote_time_cost)) + + # create database replica 3 vgroups 1 + + db1 = 'db_1' + create_db_replica_3_vgroups_1 = "create database {} replica 3 vgroups 1".format(db1) + tdLog.info('=======database {} replica 3 vgroups 1 ======'.format(db1)) + tdSql.execute(create_db_replica_3_vgroups_1) + self.vote_leader_time_costs(db1) + self.exec_revote_action(db1) + + # create database replica 3 vgroups 10 + db2 = 'db_2' + create_db_replica_3_vgroups_10 = "create database {} replica 3 vgroups 10".format(db2) + tdLog.info('=======database {} replica 3 vgroups 10 ======'.format(db2)) + tdSql.execute(create_db_replica_3_vgroups_10) + self.vote_leader_time_costs(db2) + self.exec_revote_action(db2) + + # # create database replica 3 vgroups 100 + # db3 = 'db_3' + # create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 100".format(db3) + # tdLog.info('=======database {} replica 3 vgroups 100 ======'.format(db3)) + # tdSql.execute(create_db_replica_3_vgroups_100) + # self.vote_leader_time_costs(db3) + # self.exec_revote_action(db3) + + + + def run(self): + self.check_setup_cluster_status() + self.test_init_vgroups_time_costs() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 2fa173c0471fc6aaab22516678b6120d80aad9eb Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 21 Jul 2022 16:06:07 +0800 Subject: [PATCH 04/36] revote leader when stop one dnode --- .../4dnode1mnode_basic_replica3_vgroups_stopOne.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py index 3244b6bd7b..3be36c067e 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups_stopOne.py @@ -339,13 +339,13 @@ class TDTestCase: self.vote_leader_time_costs(db2) self.exec_revote_action(db2) - # # create database replica 3 vgroups 100 - # db3 = 'db_3' - # create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 100".format(db3) - # tdLog.info('=======database {} replica 3 vgroups 100 ======'.format(db3)) - # tdSql.execute(create_db_replica_3_vgroups_100) - # self.vote_leader_time_costs(db3) - # self.exec_revote_action(db3) + # create database replica 3 vgroups 100 + db3 = 'db_3' + create_db_replica_3_vgroups_100 = "create database {} replica 3 vgroups 100".format(db3) + tdLog.info('=======database {} replica 3 vgroups 100 ======'.format(db3)) + tdSql.execute(create_db_replica_3_vgroups_100) + self.vote_leader_time_costs(db3) + self.exec_revote_action(db3) From ce900e4835cf63e36f314080f3981222943270b1 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Thu, 21 Jul 2022 19:58:28 +0800 Subject: [PATCH 05/36] add case of stop follow of vnode --- tests/pytest/util/common.py | 4 +- ...asic_replica3_insertdatas_stop_follower.py | 368 ++++++++++++++++++ 2 files changed, 370 insertions(+), 2 deletions(-) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower.py diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 94043ed01a..e7655881b8 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -382,14 +382,14 @@ class TDCom: def newcon(self,host='localhost',port=6030,user='root',password='taosdata'): con=taos.connect(host=host, user=user, password=password, port=port) - print(con) + # print(con) return con def newcur(self,host='localhost',port=6030,user='root',password='taosdata'): cfgPath = self.getClientCfgPath() con=taos.connect(host=host, user=user, password=password, config=cfgPath, port=port) cur=con.cursor() - print(cur) + # print(cur) return cur def newTdSql(self, host='localhost',port=6030,user='root',password='taosdata'): diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower.py new file mode 100644 index 0000000000..63d560587d --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower.py @@ -0,0 +1,368 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 5 + self.current_thread = None + self.max_restart_time = 20 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + tdSql.checkData(0 , 0 , tb_nums*row_nums+append_rows) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + tdSql.checkRows(tb_nums) + + def _get_stop_dnode_id(self,dbname): + tdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = tdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='follower': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin stop dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + self.wait_stop_dnode_OK() + + # append rows of stablename when dnode stop + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin start dnode + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + start = time.time() + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + tdDnodes[self.stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.sync_run_case() + self.unsync_run_case() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 6f4851a6a2ea105f9c26a188cd3193073fa88cca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 14:38:28 +0800 Subject: [PATCH 06/36] refactor: do some internal refactor. --- source/libs/executor/inc/executil.h | 3 + source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executil.c | 76 +++++ source/libs/executor/src/executor.c | 417 +++++++++++++++++++++++- source/libs/executor/src/executorMain.c | 405 ----------------------- source/libs/executor/src/executorimpl.c | 86 +---- source/libs/executor/src/scanoperator.c | 2 +- 7 files changed, 508 insertions(+), 482 deletions(-) delete mode 100644 source/libs/executor/src/executorMain.c diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 625541e00a..62ce56ee91 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -108,6 +108,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); +int32_t getGroupIdFromTableTags(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +size_t getTableTagsBufLen(const SNodeList* pGroups); + SArray* createSortInfo(SNodeList* pNodeList); SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 21068c68a4..3d76a03647 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -423,6 +423,7 @@ typedef struct SStreamScanInfo { // status for tmq // SSchemaWrapper schema; STqOffset offset; + SNodeList* pGroupTags; SNode* pTagCond; SNode* pTagIndexCond; } SStreamScanInfo; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 18bb8a57f4..1a34635fe8 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -379,6 +379,82 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, return code; } +size_t getTableTagsBufLen(const SNodeList* pGroups) { + size_t keyLen = 0; + + SNode* node; + FOREACH(node, pGroups) { + SExprNode* pExpr = (SExprNode*)node; + keyLen += pExpr->resType.bytes; + } + + keyLen += sizeof(int8_t) * LIST_LENGTH(pGroups); + return keyLen; +} + +int32_t getGroupIdFromTableTags(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { + SMetaReader mr = {0}; + metaReaderInit(&mr, pMeta, 0); + metaGetTableEntryByUid(&mr, uid); + + SNodeList* groupNew = nodesCloneList(pGroupNode); + + nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); + char* isNull = (char*)keyBuf; + char* pStart = (char*)keyBuf + sizeof(int8_t)*LIST_LENGTH(pGroupNode); + + SNode* pNode; + int32_t index = 0; + FOREACH(pNode, groupNew) { + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pNew); + } else { + taosMemoryFree(keyBuf); + nodesDestroyList(groupNew); + metaReaderClear(&mr); + return code; + } + + ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); + SValueNode* pValue = (SValueNode*)pNew; + + if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) { + isNull[index++] = 1; + continue; + } else { + isNull[index++] = 0; + char* data = nodesGetValueFromNode(pValue); + if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { + if (tTagIsJson(data)) { + terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; + taosMemoryFree(keyBuf); + nodesDestroyList(groupNew); + metaReaderClear(&mr); + return terrno; + } + int32_t len = getJsonValueLen(data); + memcpy(pStart, data, len); + pStart += len; + } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + memcpy(pStart, data, varDataTLen(data)); + pStart += varDataTLen(data); + } else { + memcpy(pStart, data, pValue->node.resType.bytes); + pStart += pValue->node.resType.bytes; + } + } + } + + int32_t len = (int32_t)(pStart - (char*)keyBuf); + *pGroupId = calcGroupId(keyBuf, len); + + nodesDestroyList(groupNew); + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; +} + SArray* extractPartitionColInfo(SNodeList* pNodeList) { if (!pNodeList) { return NULL; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index b00dc9dba5..0d547c2a14 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,10 +14,21 @@ */ #include "executor.h" +#include "tref.h" #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" #include "vnode.h" +#include "tudf.h" + +static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; +int32_t exchangeObjRefPool = -1; + +static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); } +static void cleanupRefPool() { + int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0); + taosCloseRef(ref); +} static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid, char* id) { @@ -184,8 +195,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers) { return pTaskInfo; } -static SArray* filterQualifiedChildTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, - const char* idstr) { +static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const SArray* tableIdList, const char* idstr) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); // let's discard the tables those are not created according to the queried super table. @@ -239,7 +249,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo int32_t code = 0; SStreamScanInfo* pScanInfo = pInfo->info; if (isAdd) { // add new table id - SArray* qa = filterQualifiedChildTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); qDebug(" %d qualified child tables added into stream scanner", (int32_t)taosArrayGetSize(qa)); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); @@ -247,15 +257,35 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo return code; } - // add to qTaskInfo // todo refactor STableList - for(int32_t i = 0; i < taosArrayGetSize(qa); ++i) { - uint64_t* uid = taosArrayGet(qa, i); + size_t bufLen = (pScanInfo->pGroupTags != NULL)? getTableTagsBufLen(pScanInfo->pGroupTags):0; + char* keyBuf = NULL; + if (bufLen > 0) { + keyBuf = taosMemoryMalloc(bufLen); + if (keyBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + for(int32_t i = 0; i < taosArrayGetSize(qa); ++i) { + uint64_t* uid = taosArrayGet(qa, i); STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0}; + + if (bufLen > 0) { + code = getGroupIdFromTableTags(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, + &keyInfo.groupId); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); } + if (keyBuf != NULL) { + taosMemoryFree(keyBuf); + } + taosArrayDestroy(qa); } else { // remove the table id in current list qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); @@ -289,3 +319,378 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table return 0; } + +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) { + assert(pSubplan != NULL); + SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; + + taosThreadOnce(&initPoolOnce, initRefPool); + atexit(cleanupRefPool); + + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; + code = dsDataSinkMgtInit(&cfg); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + if (handle) { + void* pSinkParam = NULL; + code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam); + } + + _error: + // if failed to add ref for all tables in this query, abort current query + return code; +} + +#ifdef TEST_IMPL +// wait moment +int waitMoment(SQInfo* pQInfo) { + if (pQInfo->sql) { + int ms = 0; + char* pcnt = strstr(pQInfo->sql, " count(*)"); + if (pcnt) return 0; + + char* pos = strstr(pQInfo->sql, " t_"); + if (pos) { + pos += 3; + ms = atoi(pos); + while (*pos >= '0' && *pos <= '9') { + pos++; + } + char unit_char = *pos; + if (unit_char == 'h') { + ms *= 3600 * 1000; + } else if (unit_char == 'm') { + ms *= 60 * 1000; + } else if (unit_char == 's') { + ms *= 1000; + } + } + if (ms == 0) return 0; + printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); + + if (ms < 1000) { + taosMsleep(ms); + } else { + int used_ms = 0; + while (used_ms < ms) { + taosMsleep(1000); + used_ms += 1000; + if (isTaskKilled(pQInfo)) { + printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); + break; + } + } + } + } + return 1; +} +#endif + +int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); + + *pRes = NULL; + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; + return pTaskInfo->code; + } + + if (pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); + } + + if (isTaskKilled(pTaskInfo)) { + qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); + return TSDB_CODE_SUCCESS; + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pTaskInfo->env); + if (ret != TSDB_CODE_SUCCESS) { + pTaskInfo->code = ret; + cleanUpUdfs(); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + atomic_store_64(&pTaskInfo->owner, 0); + + return pTaskInfo->code; + } + + qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); + + int64_t st = taosGetTimestampUs(); + + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); + uint64_t el = (taosGetTimestampUs() - st); + + pTaskInfo->cost.elapsedTime += el; + if (NULL == *pRes) { + *useconds = pTaskInfo->cost.elapsedTime; + } + + cleanUpUdfs(); + + int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; + uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; + + qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); + + atomic_store_64(&pTaskInfo->owner, 0); + return pTaskInfo->code; +} + +int32_t qKillTask(qTaskInfo_t qinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; + + if (pTaskInfo == NULL) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("%s execTask killed", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while (pTaskInfo->owner != 0) { + taosMsleep(100); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qAsyncKillTask(qTaskInfo_t qinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; + + if (pTaskInfo == NULL) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo); + + return TSDB_CODE_SUCCESS; +} + +void qDestroyTask(qTaskInfo_t qTaskHandle) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle; + if (pTaskInfo == NULL) { + return; + } + + qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows); + + queryCostStatis(pTaskInfo); // print the query cost summary + doDestroyTask(pTaskInfo); +} + +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int32_t capacity = 0; + + return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); +} + +int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { + SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; + if (pTaskInfo->pRoot == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + int32_t nOptrWithVal = 0; + int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); + if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { + taosMemoryFreeClear(*pOutput); + *len = 0; + } + return code; +} + +int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { + SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; + + if (pTaskInfo == NULL || pInput == NULL || len == 0) { + return TSDB_CODE_INVALID_PARA; + } + + return decodeOperator(pTaskInfo->pRoot, pInput, len); +} + +int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + + while (1) { + uint8_t type = pOperator->operatorType; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + *scanner = pOperator->info; + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } +} + +#if 0 +int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + taosWriteQitem(pTaskInfo->streamInfo.inputQueue->queue, pItem); + return 0; +} +#endif + +int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + pTaskInfo->streamInfo.recoverStartVer = startVer; + pTaskInfo->streamInfo.recoverEndVer = endVer; + pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE; + return 0; +} + +void* qExtractReaderFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return (void*)pInfo->tqReader; +} + +const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return pInfo->tqReader->pSchemaWrapper; +} + +const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { + SStreamScanInfo* pInfo = scanner; + return &pInfo->offset; +} + +void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + return pTaskInfo->streamInfo.metaBlk; +} + +int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); + return 0; +} + +int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + pTaskInfo->streamInfo.prepareStatus = *pOffset; + if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { + while (1) { + uint8_t type = pOperator->operatorType; + pOperator->status = OP_OPENED; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pOperator->info; + if (pOffset->type == TMQ_OFFSET__LOG) { +#if 0 + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && + pInfo->tqReader->pWalReader->curVersion != pOffset->version) { + qError("prepare scan ver %ld actual ver %ld, last %ld", pOffset->version, + pInfo->tqReader->pWalReader->curVersion, pTaskInfo->streamInfo.lastStatus.version); + ASSERT(0); + } +#endif + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) { + return -1; + } + ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1); + } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { + /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/ + int64_t uid = pOffset->uid; + int64_t ts = pOffset->ts; + + if (uid == 0) { + if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); + uid = pTableInfo->uid; + ts = INT64_MIN; + } else { + return -1; + } + } + /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ + /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); + bool found = false; + for (int32_t i = 0; i < tableSz; i++) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); + if (pTableInfo->uid == uid) { + found = true; + pTableScanInfo->currentTable = i; + break; + } + } + + // TODO after dropping table, table may be not found + ASSERT(found); + + tsdbSetTableId(pTableScanInfo->dataReader, uid); + int64_t oldSkey = pTableScanInfo->cond.twindows.skey; + pTableScanInfo->cond.twindows.skey = ts + 1; + tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); + pTableScanInfo->cond.twindows.skey = oldSkey; + pTableScanInfo->scanTimes = 0; + + qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, + pTableScanInfo->currentTable, tableSz); + /*}*/ + + } else { + ASSERT(0); + } + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } + } + return 0; +} + +#if 0 +int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + + if (uid == 0) { + if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); + uid = pTableInfo->uid; + ts = INT64_MIN; + } + } + + return doPrepareScan(pTaskInfo->pRoot, uid, ts); +} + +int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + + return doGetScanStatus(pTaskInfo->pRoot, uid, ts); +} +#endif + diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c deleted file mode 100644 index e0020a496e..0000000000 --- a/source/libs/executor/src/executorMain.c +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "dataSinkMgt.h" -#include "os.h" -#include "tmsg.h" -#include "tref.h" -#include "tudf.h" - -#include "executor.h" -#include "executorimpl.h" -#include "query.h" - -static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; -int32_t exchangeObjRefPool = -1; - -static void initRefPool() { exchangeObjRefPool = taosOpenRef(1024, doDestroyExchangeOperatorInfo); } -static void cleanupRefPool() { - int32_t ref = atomic_val_compare_exchange_32(&exchangeObjRefPool, exchangeObjRefPool, 0); - taosCloseRef(ref); -} - -int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, const char* sql, EOPTR_EXEC_MODEL model) { - assert(pSubplan != NULL); - SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - - taosThreadOnce(&initPoolOnce, initRefPool); - atexit(cleanupRefPool); - - int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, sql, model); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; - code = dsDataSinkMgtInit(&cfg); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - if (handle) { - void* pSinkParam = NULL; - code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTaskInfo, readHandle); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam); - } - -_error: - // if failed to add ref for all tables in this query, abort current query - return code; -} - -#ifdef TEST_IMPL -// wait moment -int waitMoment(SQInfo* pQInfo) { - if (pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if (pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if (pos) { - pos += 3; - ms = atoi(pos); - while (*pos >= '0' && *pos <= '9') { - pos++; - } - char unit_char = *pos; - if (unit_char == 'h') { - ms *= 3600 * 1000; - } else if (unit_char == 'm') { - ms *= 60 * 1000; - } else if (unit_char == 's') { - ms *= 1000; - } - } - if (ms == 0) return 0; - printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if (ms < 1000) { - taosMsleep(ms); - } else { - int used_ms = 0; - while (used_ms < ms) { - taosMsleep(1000); - used_ms += 1000; - if (isTaskKilled(pQInfo)) { - printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); - break; - } - } - } - } - return 1; -} -#endif - -int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int64_t threadId = taosGetSelfPthreadId(); - - *pRes = NULL; - int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { - qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); - pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; - return pTaskInfo->code; - } - - if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); - } - - if (isTaskKilled(pTaskInfo)) { - qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); - return TSDB_CODE_SUCCESS; - } - - // error occurs, record the error code and return to client - int32_t ret = setjmp(pTaskInfo->env); - if (ret != TSDB_CODE_SUCCESS) { - pTaskInfo->code = ret; - cleanUpUdfs(); - qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); - return pTaskInfo->code; - } - - qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); - - int64_t st = taosGetTimestampUs(); - - *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot); - uint64_t el = (taosGetTimestampUs() - st); - - pTaskInfo->cost.elapsedTime += el; - if (NULL == *pRes) { - *useconds = pTaskInfo->cost.elapsedTime; - } - - cleanUpUdfs(); - - int32_t current = (*pRes != NULL) ? (*pRes)->info.rows : 0; - uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; - - qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", - GET_TASKID(pTaskInfo), current, total, 0, el / 1000.0); - - atomic_store_64(&pTaskInfo->owner, 0); - return pTaskInfo->code; -} - -int32_t qKillTask(qTaskInfo_t qinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; - - if (pTaskInfo == NULL) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - qDebug("%s execTask killed", GET_TASKID(pTaskInfo)); - setTaskKilled(pTaskInfo); - - // Wait for the query executing thread being stopped/ - // Once the query is stopped, the owner of qHandle will be cleared immediately. - while (pTaskInfo->owner != 0) { - taosMsleep(100); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t qAsyncKillTask(qTaskInfo_t qinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; - - if (pTaskInfo == NULL) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); - setTaskKilled(pTaskInfo); - - return TSDB_CODE_SUCCESS; -} - -void qDestroyTask(qTaskInfo_t qTaskHandle) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle; - if (pTaskInfo == NULL) { - return; - } - - qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows); - - queryCostStatis(pTaskInfo); // print the query cost summary - doDestroyTask(pTaskInfo); -} - -int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, int32_t* resNum, SExplainExecInfo** pRes) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t capacity = 0; - - return getOperatorExplainExecInfo(pTaskInfo->pRoot, pRes, &capacity, resNum); -} - -int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { - SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; - if (pTaskInfo->pRoot == NULL) { - return TSDB_CODE_INVALID_PARA; - } - - int32_t nOptrWithVal = 0; - int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal = 0)) { - taosMemoryFreeClear(*pOutput); - *len = 0; - } - return code; -} - -int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { - SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; - - if (pTaskInfo == NULL || pInput == NULL || len == 0) { - return TSDB_CODE_INVALID_PARA; - } - - return decodeOperator(pTaskInfo->pRoot, pInput, len); -} - -int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - SOperatorInfo* pOperator = pTaskInfo->pRoot; - - while (1) { - uint8_t type = pOperator->operatorType; - if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - *scanner = pOperator->info; - return 0; - } else { - ASSERT(pOperator->numOfDownstream == 1); - pOperator = pOperator->pDownstream[0]; - } - } -} - -#if 0 -int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - taosWriteQitem(pTaskInfo->streamInfo.inputQueue->queue, pItem); - return 0; -} -#endif - -int32_t qStreamPrepareRecover(qTaskInfo_t tinfo, int64_t startVer, int64_t endVer) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.recoverStartVer = startVer; - pTaskInfo->streamInfo.recoverEndVer = endVer; - pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE; - return 0; -} - -void* qExtractReaderFromStreamScanner(void* scanner) { - SStreamScanInfo* pInfo = scanner; - return (void*)pInfo->tqReader; -} - -const SSchemaWrapper* qExtractSchemaFromStreamScanner(void* scanner) { - SStreamScanInfo* pInfo = scanner; - return pInfo->tqReader->pSchemaWrapper; -} - -const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { - SStreamScanInfo* pInfo = scanner; - return &pInfo->offset; -} - -void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - return pTaskInfo->streamInfo.metaBlk; -} - -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); - return 0; -} - -int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - SOperatorInfo* pOperator = pTaskInfo->pRoot; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); - pTaskInfo->streamInfo.prepareStatus = *pOffset; - if (!tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { - while (1) { - uint8_t type = pOperator->operatorType; - pOperator->status = OP_OPENED; - if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamScanInfo* pInfo = pOperator->info; - if (pOffset->type == TMQ_OFFSET__LOG) { -#if 0 - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus) && - pInfo->tqReader->pWalReader->curVersion != pOffset->version) { - qError("prepare scan ver %ld actual ver %ld, last %ld", pOffset->version, - pInfo->tqReader->pWalReader->curVersion, pTaskInfo->streamInfo.lastStatus.version); - ASSERT(0); - } -#endif - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1) < 0) { - return -1; - } - ASSERT(pInfo->tqReader->pWalReader->curVersion == pOffset->version + 1); - } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - /*pInfo->blockType = STREAM_INPUT__TABLE_SCAN;*/ - int64_t uid = pOffset->uid; - int64_t ts = pOffset->ts; - - if (uid == 0) { - if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); - uid = pTableInfo->uid; - ts = INT64_MIN; - } else { - return -1; - } - } - /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ - /*pTaskInfo->streamInfo.lastStatus.uid != uid || pTaskInfo->streamInfo.lastStatus.ts != ts) {*/ - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); - bool found = false; - for (int32_t i = 0; i < tableSz; i++) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i); - if (pTableInfo->uid == uid) { - found = true; - pTableScanInfo->currentTable = i; - break; - } - } - - // TODO after dropping table, table may be not found - ASSERT(found); - - tsdbSetTableId(pTableScanInfo->dataReader, uid); - int64_t oldSkey = pTableScanInfo->cond.twindows.skey; - pTableScanInfo->cond.twindows.skey = ts + 1; - tsdbReaderReset(pTableScanInfo->dataReader, &pTableScanInfo->cond); - pTableScanInfo->cond.twindows.skey = oldSkey; - pTableScanInfo->scanTimes = 0; - - qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, - pTableScanInfo->currentTable, tableSz); - /*}*/ - - } else { - ASSERT(0); - } - return 0; - } else { - ASSERT(pOperator->numOfDownstream == 1); - pOperator = pOperator->pDownstream[0]; - } - } - } - return 0; -} - -#if 0 -int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - - if (uid == 0) { - if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) { - STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); - uid = pTableInfo->uid; - ts = INT64_MIN; - } - } - - return doPrepareScan(pTaskInfo->pRoot, uid, ts); -} - -int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - - return doGetScanStatus(pTaskInfo->pRoot, uid, ts); -} -#endif diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a930a7bb46..d3bfac82b6 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3187,7 +3187,8 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { return TDB_CODE_SUCCESS; } -int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SSDataBlock** pExistedBlock, + bool holdDataInBuf) { if (pLimitInfo->remainGroupOffset > 0) { if (pLimitInfo->currentGroupId == 0) { // it is the first group pLimitInfo->currentGroupId = pBlock->info.groupId; @@ -3208,6 +3209,7 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa pLimitInfo->currentGroupId = pBlock->info.groupId; } + // here check for a new group data, we need to handle the data of the previous group. if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { pLimitInfo->numOfOutputGroups += 1; if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { @@ -3220,6 +3222,13 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa // reset the value for a new group data pLimitInfo->numOfOutputRows = 0; pLimitInfo->remainOffset = pLimitInfo->limit.offset; + + *pExistedBlock = pBlock; + + // existing rows that belongs to previous group. + if (pBlock->info.rows > 0) { + return PROJECT_RETRIEVE_DONE; + } } // here we reach the start position, according to the limit/offset requirements. @@ -3305,18 +3314,12 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { while (1) { // The downstream exec may change the value of the newgroup, so use a local variable instead. - qDebug("projection call next"); SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - qDebug("projection get null"); - - /*if (pTaskInfo->execModel == OPTR_EXEC_MODEL_BATCH) {*/ doSetOperatorCompleted(pOperator); - /*} else if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) {*/ - /*pOperator->status = OP_RES_TO_RETURN;*/ - /*}*/ break; } + if (pBlock->info.type == STREAM_RETRIEVE) { // for stream interval return pBlock; @@ -4133,9 +4136,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT return pTaskInfo; } -static STsdbReader* doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, const char* idstr); - static SArray* extractColumnInfo(SNodeList* pNodeList); SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); @@ -4292,69 +4292,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t groupNum = 0; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->meta, 0); - metaGetTableEntryByUid(&mr, info->uid); - - SNodeList* groupNew = nodesCloneList(group); - - nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); - char* isNull = (char*)keyBuf; - char* pStart = (char*)keyBuf + nullFlagSize; - - SNode* pNode; - int32_t index = 0; - FOREACH(pNode, groupNew) { - SNode* pNew = NULL; - int32_t code = scalarCalculateConstants(pNode, &pNew); - if (TSDB_CODE_SUCCESS == code) { - REPLACE_NODE(pNew); - } else { - taosMemoryFree(keyBuf); - nodesDestroyList(groupNew); - metaReaderClear(&mr); - return code; - } - - ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); - SValueNode* pValue = (SValueNode*)pNew; - - if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) { - isNull[index++] = 1; - continue; - } else { - isNull[index++] = 0; - char* data = nodesGetValueFromNode(pValue); - if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { - if (tTagIsJson(data)) { - terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; - taosMemoryFree(keyBuf); - nodesDestroyList(groupNew); - metaReaderClear(&mr); - return terrno; - } - int32_t len = getJsonValueLen(data); - memcpy(pStart, data, len); - pStart += len; - } else if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { - memcpy(pStart, data, varDataTLen(data)); - pStart += varDataTLen(data); - } else { - memcpy(pStart, data, pValue->node.resType.bytes); - pStart += pValue->node.resType.bytes; - } - } + int32_t code = getGroupIdFromTableTags(pHandle->meta, info->uid, group, keyBuf, &info->groupId); + if (code != TSDB_CODE_SUCCESS) { + return code; } - int32_t len = (int32_t)(pStart - (char*)keyBuf); - uint64_t groupId = calcGroupId(keyBuf, len); - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); - info->groupId = groupId; + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &info->groupId, sizeof(uint64_t)); groupNum++; - - nodesDestroyList(groupNew); - metaReaderClear(&mr); } + taosMemoryFree(keyBuf); if (pTableListInfo->needSortTableByGroupId) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ab62905c3f..2c206fbc12 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1538,7 +1538,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; pInfo->pTagCond = pTagCond; - + pInfo->pGroupTags = pTableScanNode->pGroupTags; pInfo->twAggSup = *pTwSup; int32_t numOfCols = 0; From 7cce7da526801eaee99a8b39837ac6058dfc9b6e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 22 Jul 2022 16:29:22 +0800 Subject: [PATCH 07/36] add test case stop follower and leader --- ...4dnode1mnode_basic_replica3_insertdatas.py | 179 ++++++ ...replica3_insertdatas_stop_follower_sync.py | 484 ++++++++++++++++ ...lica3_insertdatas_stop_follower_unsync.py} | 126 +++- ..._basic_replica3_insertdatas_stop_leader.py | 548 ++++++++++++++++++ 4 files changed, 1332 insertions(+), 5 deletions(-) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py rename tests/system-test/6-cluster/vnode/{4dnode1mnode_basic_replica3_insertdatas_stop_follower.py => 4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py} (71%) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py new file mode 100644 index 0000000000..00bd8a48d9 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py @@ -0,0 +1,179 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 2 + self.tb_nums = 10 + self.row_nums = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_db_replica_3_insertdatas(self, dbname, replica_num ,vgroup_nums ,tb_nums , row_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(tb_nums): + sub_tbname = "sub_tb_{}".format(i) + tdSql.execute("create table {} using stb1 tags({})".format(sub_tbname,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + 1000*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== create database {} and insert rows execute end =====".format(dbname)) + + def check_insert_status(self, dbname, tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.query("select count(*) from {}.{}".format(dbname,'stb1')) + tdSql.checkData(0 , 0 , tb_nums*row_nums) + tdSql.query("select distinct tbname from {}.{}".format(dbname,'stb1')) + tdSql.checkRows(tb_nums) + + def run(self): + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.create_db_replica_3_insertdatas(self.db_name , self.replica , self.vgroups , self.tb_nums , self.row_nums) + self.check_insert_status(self.db_name , self.tb_nums , self.row_nums) + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py new file mode 100644 index 0000000000..5db1ced199 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py @@ -0,0 +1,484 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import datetime +import inspect +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 5 + self.current_thread = None + self.max_restart_time = 10 + self.try_check_times = 10 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + + def _get_stop_dnode_id(self,dbname): + tdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = tdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='follower': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin stop dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + self.wait_stop_dnode_OK() + + # append rows of stablename when dnode stop + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin start dnode + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + start = time.time() + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + tdDnodes[self.stop_dnode_id-1].stoptaosd() + self.wait_stop_dnode_OK() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + self.sync_run_case() + # self.unsync_run_case() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py similarity index 71% rename from tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower.py rename to tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py index 63d560587d..854a28b48a 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py @@ -12,6 +12,8 @@ from util.dnodes import TDDnodes from util.dnodes import TDDnode from util.cluster import * +import datetime +import inspect import time import socket import subprocess @@ -35,7 +37,8 @@ class TDTestCase: self.stop_dnode_id = None self.loop_restart_times = 5 self.current_thread = None - self.max_restart_time = 20 + self.max_restart_time = 10 + self.try_check_times = 10 def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -173,10 +176,37 @@ class TDTestCase: def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): tdSql.execute("use {}".format(dbname)) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) - tdSql.checkData(0 , 0 , tb_nums*row_nums+append_rows) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) - tdSql.checkRows(tb_nums) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 def _get_stop_dnode_id(self,dbname): tdSql.query("show {}.vgroups".format(dbname)) @@ -238,6 +268,92 @@ class TDTestCase: # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + def sync_run_case(self): # stop follower and insert datas , update tables and create new stables tdDnodes=cluster.dnodes @@ -251,7 +367,7 @@ class TDTestCase: # check rows of datas self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) - + # begin stop dnode start = time.time() tdDnodes[self.stop_dnode_id-1].stoptaosd() @@ -354,7 +470,7 @@ class TDTestCase: # basic insert and check of cluster self.check_setup_cluster_status() self.create_db_check_vgroups() - self.sync_run_case() + # self.sync_run_case() self.unsync_run_case() diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py new file mode 100644 index 0000000000..9dde4a6382 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -0,0 +1,548 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 50 + self.current_thread = None + self.max_restart_time = 5 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + + def _get_stop_dnode_id(self,dbname): + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='leader': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def get_leader_infos(self ,dbname): + + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + + leader_infos = set() + for vgroup_info in vgroup_infos: + leader_infos.add(vgroup_info[3:-4]) + + return leader_infos + + def check_revote_leader_success(self, dbname, before_leader_infos , after_leader_infos): + check_status = False + vote_act = set(set(after_leader_infos)-set(before_leader_infos)) + if not vote_act: + print("=======before_revote_leader_infos ======\n" , before_leader_infos) + print("=======after_revote_leader_infos ======\n" , after_leader_infos) + tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + else: + for vgroup_info in vote_act: + for ind , role in enumerate(vgroup_info): + if role==self.stop_dnode_id: + + if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: + tdLog.info(" === revote leader ok , leader is {} now ====".format(list(vgroup_info).index("leader")-1)) + check_status = True + elif vgroup_info[ind+1] !="offline": + tdLog.info(" === dnode {} should be offline ".format(self.stop_dnode_id)) + else: + continue + break + return check_status + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # get leader info before stop + before_leader_infos = self.get_leader_infos(db_name) + + # begin stop dnode + + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + self.wait_stop_dnode_OK() + + # vote leaders check + + # get leader info after stop + after_leader_infos = self.get_leader_infos(db_name) + + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + # append rows of stablename when dnode stop make sure revote leaders + + while not revote_status: + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + + if revote_status: + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + else: + tdLog.info("===== leader of database {} is not ok , append rows fail =====".format(db_name)) + + # begin start dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + + # get leader info before stop + before_leader_infos = self.get_leader_infos(db_name) + + tdDnodes[self.stop_dnode_id-1].stoptaosd() + + # self.wait_stop_dnode_OK() + + # check revote leader when restart servers + # # get leader info after stop + # after_leader_infos = self.get_leader_infos(db_name) + # revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + # # append rows of stablename when dnode stop make sure revote leaders + # while not revote_status: + # after_leader_infos = self.get_leader_infos(db_name) + # revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + tdDnodes[self.stop_dnode_id-1].starttaosd() + start = time.time() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + # self.sync_run_case() + self.unsync_run_case() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From cdf1a72f7ca6c75cd0656b7159249bc6e3073320 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 22 Jul 2022 16:51:29 +0800 Subject: [PATCH 08/36] add test case of stop leader of an database --- ..._basic_replica3_insertdatas_stop_leader.py | 49 +++++++++---------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index 9dde4a6382..d326a23137 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -33,7 +33,7 @@ class TDTestCase: self.tb_nums = 10 self.row_nums = 100 self.stop_dnode_id = None - self.loop_restart_times = 50 + self.loop_restart_times = 10 self.current_thread = None self.max_restart_time = 5 @@ -459,12 +459,29 @@ class TDTestCase: tdDnodes=cluster.dnodes self.stop_dnode_id = self._get_stop_dnode_id(dbname) # begin restart dnode - - # get leader info before stop - before_leader_infos = self.get_leader_infos(db_name) tdDnodes[self.stop_dnode_id-1].stoptaosd() + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # # get leader info before stop + # before_leader_infos = self.get_leader_infos(db_name) # self.wait_stop_dnode_OK() # check revote leader when restart servers @@ -508,23 +525,6 @@ class TDTestCase: # check rows of datas self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) - tbname = "sub_{}_{}".format(stablename , 0) - tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) - self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) - tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) - self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) - - # create new stables - tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) - self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) - tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) - self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) - - # create new stables again - tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) - self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) - tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) - self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) self.current_thread.join() @@ -534,11 +534,8 @@ class TDTestCase: # basic insert and check of cluster self.check_setup_cluster_status() self.create_db_check_vgroups() - # self.sync_run_case() - self.unsync_run_case() - - - + self.sync_run_case() + # self.unsync_run_case() def stop(self): tdSql.close() From f9c907bd59e72ae1cfa69effe3a3acd5ff371a80 Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 22 Jul 2022 18:47:23 +0800 Subject: [PATCH 09/36] add test case for stop follower by kill -9 --- ...replica3_insertdatas_stop_follower_sync.py | 16 +- ...plica3_insertdatas_stop_follower_unsync.py | 17 +- ...rtdatas_stop_follower_unsync_force_stop.py | 521 ++++++++++++++++++ ..._basic_replica3_insertdatas_stop_leader.py | 17 +- 4 files changed, 565 insertions(+), 6 deletions(-) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py index 5db1ced199..22d3ba6dbc 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_sync.py @@ -179,22 +179,32 @@ class TDTestCase: tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) count = 0 while not status_OK : if count > self.try_check_times: - os.system("taos -s ' show {}.vgroups;'".format(dbname)) + os.system("taos -s ' show {}.vgroups; '".format(dbname)) tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) count += 1 tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) count = 0 while not status_OK : @@ -204,10 +214,12 @@ class TDTestCase: break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) count += 1 - def _get_stop_dnode_id(self,dbname): tdSql.query("show {}.vgroups".format(dbname)) vgroup_infos = tdSql.queryResult diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py index 854a28b48a..28198b9529 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync.py @@ -174,11 +174,15 @@ class TDTestCase: os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): - + tdSql.execute("use {}".format(dbname)) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) count = 0 @@ -189,12 +193,18 @@ class TDTestCase: break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) count += 1 tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) count = 0 while not status_OK : @@ -204,10 +214,13 @@ class TDTestCase: break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) count += 1 - + def _get_stop_dnode_id(self,dbname): tdSql.query("show {}.vgroups".format(dbname)) vgroup_infos = tdSql.queryResult diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py new file mode 100644 index 0000000000..83faba4578 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_follower_unsync_force_stop.py @@ -0,0 +1,521 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import datetime +import inspect +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 5 + self.current_thread = None + self.max_restart_time = 10 + self.try_check_times = 10 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + + def _get_stop_dnode_id(self,dbname): + tdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = tdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='follower': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin stop dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].forcestop() + + self.wait_stop_dnode_OK() + + # append rows of stablename when dnode stop + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # begin start dnode + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + start = time.time() + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + while not self.stop_dnode_id: + time.sleep(0.5) + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + + # force stop taosd by kill -9 + self.force_stop_dnode(self.stop_dnode_id) + self.wait_stop_dnode_OK() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + self.current_thread.join() + + def force_stop_dnode(self, dnode_id ): + + tdSql.query("show dnodes") + port = None + for dnode_info in tdSql.queryResult: + if dnode_id == dnode_info[0]: + port = dnode_info[1].split(":")[-1] + break + else: + continue + if port: + tdLog.info(" ==== dnode {} will be force stop by kill -9 ====".format(dnode_id)) + psCmd = '''netstat -anp|grep -w LISTEN|grep -w %s |grep -o "LISTEN.*"|awk '{print $2}'|cut -d/ -f1|head -n1''' %(port) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + ps_kill_taosd = ''' kill -9 {} '''.format(processID) + # print(ps_kill_taosd) + os.system(ps_kill_taosd) + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + # self.sync_run_case() + self.unsync_run_case() + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index d326a23137..47f0ffb61f 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -262,21 +262,32 @@ class TDTestCase: tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) count = 0 while not status_OK : if count > self.try_check_times: - os.system("taos -s ' show {}.vgroups;'".format(dbname)) + os.system("taos -s ' show {}.vgroups; '".format(dbname)) tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) break time.sleep(0.1) tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) count += 1 + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) count = 0 while not status_OK : @@ -286,10 +297,12 @@ class TDTestCase: break time.sleep(0.1) tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) count += 1 - def _get_stop_dnode_id(self,dbname): newTdSql=tdCom.newTdSql() newTdSql.query("show {}.vgroups".format(dbname)) From 950cf5dc35f013a8ed50664d97abc0e5ad5d9c2e Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Fri, 22 Jul 2022 19:09:54 +0800 Subject: [PATCH 10/36] add case about force stop leader of cluster --- ..._basic_replica3_insertdatas_stop_leader.py | 2 +- ...ca3_insertdatas_stop_leader_forece_stop.py | 580 ++++++++++++++++++ 2 files changed, 581 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py index 47f0ffb61f..c0aafa7978 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader.py @@ -389,7 +389,7 @@ class TDTestCase: if role==self.stop_dnode_id: if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: - tdLog.info(" === revote leader ok , leader is {} now ====".format(list(vgroup_info).index("leader")-1)) + tdLog.info(" === revote leader ok , leader is {} now ====".format(vgroup_info[list(vgroup_info).index("leader")-1])) check_status = True elif vgroup_info[ind+1] !="offline": tdLog.info(" === dnode {} should be offline ".format(self.stop_dnode_id)) diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py new file mode 100644 index 0000000000..d6edfda770 --- /dev/null +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_stop_leader_forece_stop.py @@ -0,0 +1,580 @@ +# author : wenzhouwww +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * + +import time +import socket +import subprocess +import threading +sys.path.append(os.path.dirname(__file__)) + +class TDTestCase: + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.mnode_list = {} + self.dnode_list = {} + self.ts = 1483200000000 + self.ts_step =1000 + self.db_name ='testdb' + self.replica = 3 + self.vgroups = 1 + self.tb_nums = 10 + self.row_nums = 100 + self.stop_dnode_id = None + self.loop_restart_times = 10 + self.current_thread = None + self.max_restart_time = 5 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _parse_datetime(self,timestr): + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f') + except ValueError: + pass + try: + return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S') + except ValueError: + pass + + def mycheckRowCol(self, sql, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is smaller than zero" % args) + if col < 0: + args = (caller.filename, caller.lineno, sql, row) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is smaller than zero" % args) + if row > tdSql.queryRows: + args = (caller.filename, caller.lineno, sql, row, tdSql.queryRows) + tdLog.exit("%s(%d) failed: sql:%s, row:%d is larger than queryRows:%d" % args) + if col > tdSql.queryCols: + args = (caller.filename, caller.lineno, sql, col, tdSql.queryCols) + tdLog.exit("%s(%d) failed: sql:%s, col:%d is larger than queryCols:%d" % args) + + def mycheckData(self, sql ,row, col, data): + check_status = True + self.mycheckRowCol(sql ,row, col) + if tdSql.queryResult[row][col] != data: + if tdSql.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(tdSql.queryResult[row][col]) == pd.to_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%d == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + if tdSql.queryResult[row][col] == self._parse_datetime(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + + if str(tdSql.queryResult[row][col]) == str(data): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + elif isinstance(data, float) and abs(tdSql.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (sql, row, col, tdSql.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, row, col, tdSql.queryResult[row][col], data) + tdLog.info("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + + check_status = False + + if data is None: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, str): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + elif isinstance(data, float): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (sql, row, col, tdSql.queryResult[row][col], data)) + else: + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % + (sql, row, col, tdSql.queryResult[row][col], data)) + + return check_status + + def mycheckRows(self, sql, expectRows): + check_status = True + if len(tdSql.queryResult) == expectRows: + tdLog.info("sql:%s, queryRows:%d == expect:%d" % (sql, len(tdSql.queryResult), expectRows)) + return True + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, len(tdSql.queryResult), expectRows) + tdLog.info("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) + check_status = False + return check_status + + def check_setup_cluster_status(self): + tdSql.query("show mnodes") + for mnode in tdSql.queryResult: + name = mnode[1] + info = mnode + self.mnode_list[name] = info + + tdSql.query("show dnodes") + for dnode in tdSql.queryResult: + name = dnode[1] + info = dnode + self.dnode_list[name] = info + + count = 0 + is_leader = False + mnode_name = '' + for k,v in self.mnode_list.items(): + count +=1 + # only for 1 mnode + mnode_name = k + + if v[2] =='leader': + is_leader=True + + if count==1 and is_leader: + tdLog.info("===== depoly cluster success with 1 mnode as leader =====") + else: + tdLog.exit("===== depoly cluster fail with 1 mnode as leader =====") + + for k ,v in self.dnode_list.items(): + if k == mnode_name: + if v[3]==0: + tdLog.info("===== depoly cluster mnode only success at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + tdLog.exit("===== depoly cluster mnode only fail at {} , support_vnodes is {} ".format(mnode_name,v[3])) + else: + continue + + def create_db_check_vgroups(self): + + tdSql.execute("drop database if exists test") + tdSql.execute("create database if not exists test replica 1 duration 300") + tdSql.execute("use test") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + + for i in range(5): + tdSql.execute("create table sub_tb_{} using stb1 tags({})".format(i,i)) + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.query("show tables") + tdSql.checkRows(6) + + tdSql.query("show test.vgroups;") + vgroups_infos = {} # key is id: value is info list + for vgroup_info in tdSql.queryResult: + vgroup_id = vgroup_info[0] + tmp_list = [] + for role in vgroup_info[3:-4]: + if role in ['leader','follower']: + tmp_list.append(role) + vgroups_infos[vgroup_id]=tmp_list + + for k , v in vgroups_infos.items(): + if len(v) ==1 and v[0]=="leader": + tdLog.info(" === create database replica only 1 role leader check success of vgroup_id {} ======".format(k)) + else: + tdLog.exit(" === create database replica only 1 role leader check fail of vgroup_id {} ======".format(k)) + + def create_database(self, dbname, replica_num ,vgroup_nums ): + drop_db_sql = "drop database if exists {}".format(dbname) + create_db_sql = "create database {} replica {} vgroups {}".format(dbname,replica_num,vgroup_nums) + + tdLog.info(" ==== create database {} and insert rows begin =====".format(dbname)) + tdSql.execute(drop_db_sql) + tdSql.execute(create_db_sql) + tdSql.execute("use {}".format(dbname)) + + def create_stable_insert_datas(self,dbname ,stablename , tb_nums , row_nums): + tdSql.execute("use {}".format(dbname)) + tdSql.execute( + '''create table {} + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(32),c9 nchar(32), c10 timestamp) + tags (t1 int) + '''.format(stablename) + ) + + for i in range(tb_nums): + sub_tbname = "sub_{}_{}".format(stablename,i) + tdSql.execute("create table {} using {} tags({})".format(sub_tbname, stablename ,i)) + # insert datas about new database + + for row_num in range(row_nums): + ts = self.ts + self.ts_step*row_num + tdSql.execute(f"insert into {sub_tbname} values ({ts}, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + + tdLog.info(" ==== stable {} insert rows execute end =====".format(stablename)) + + def append_rows_of_exists_tables(self,dbname ,stablename , tbname , append_nums ): + + tdSql.execute("use {}".format(dbname)) + + for row_num in range(append_nums): + tdSql.execute(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + # print(f"insert into {tbname} values (now, {row_num} ,{row_num}, 10 ,1 ,{row_num} ,{row_num},true,'bin_{row_num}','nchar_{row_num}',now) ") + tdLog.info(" ==== append new rows of table {} belongs to stable {} execute end =====".format(tbname,stablename)) + os.system("taos -s 'select count(*) from {}.{}';".format(dbname,stablename)) + + def check_insert_rows(self, dbname, stablename , tb_nums , row_nums, append_rows): + + tdSql.execute("use {}".format(dbname)) + + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups; '".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select count(*) from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckData("select count(*) from {}.{}".format(dbname,stablename) ,0 , 0 , tb_nums*row_nums+append_rows) + tdLog.info(" ==== check insert rows first failed , this is {}_th retry check rows of database {}".format(count , dbname)) + count += 1 + + + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + count = 0 + while not status_OK : + if count > self.try_check_times: + os.system("taos -s ' show {}.vgroups;'".format(dbname)) + tdLog.exit(" ==== check insert rows failed after {} try check {} times of database {}".format(count , self.try_check_times ,dbname)) + break + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + while not tdSql.queryResult: + time.sleep(0.1) + tdSql.query("select distinct tbname from {}.{}".format(dbname,stablename)) + status_OK = self.mycheckRows("select distinct tbname from {}.{}".format(dbname,stablename) ,tb_nums) + tdLog.info(" ==== check insert tbnames first failed , this is {}_th retry check tbnames of database {}".format(count , dbname)) + count += 1 + + def _get_stop_dnode_id(self,dbname): + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + for vgroup_info in vgroup_infos: + leader_infos = vgroup_info[3:-4] + # print(vgroup_info) + for ind ,role in enumerate(leader_infos): + if role =='leader': + # print(ind,leader_infos) + self.stop_dnode_id = leader_infos[ind-1] + break + + + return self.stop_dnode_id + + def wait_stop_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="offline": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has stopped , id is {}".format(self.stop_dnode_id)) + + def wait_start_dnode_OK(self): + + def _get_status(): + newTdSql=tdCom.newTdSql() + status = "" + newTdSql.query("show dnodes") + dnode_infos = newTdSql.queryResult + for dnode_info in dnode_infos: + id = dnode_info[0] + dnode_status = dnode_info[4] + if id == self.stop_dnode_id: + status = dnode_status + break + return status + + status = _get_status() + while status !="ready": + time.sleep(0.1) + status = _get_status() + # tdLog.info("==== stop dnode has not been stopped , endpoint is {}".format(self.stop_dnode)) + tdLog.info("==== stop_dnode has restart , id is {}".format(self.stop_dnode_id)) + + def get_leader_infos(self ,dbname): + + newTdSql=tdCom.newTdSql() + newTdSql.query("show {}.vgroups".format(dbname)) + vgroup_infos = newTdSql.queryResult + + leader_infos = set() + for vgroup_info in vgroup_infos: + leader_infos.add(vgroup_info[3:-4]) + + return leader_infos + + def check_revote_leader_success(self, dbname, before_leader_infos , after_leader_infos): + check_status = False + vote_act = set(set(after_leader_infos)-set(before_leader_infos)) + if not vote_act: + print("=======before_revote_leader_infos ======\n" , before_leader_infos) + print("=======after_revote_leader_infos ======\n" , after_leader_infos) + tdLog.exit(" ===maybe revote not occured , there is no dnode offline ====") + else: + for vgroup_info in vote_act: + for ind , role in enumerate(vgroup_info): + if role==self.stop_dnode_id: + + if vgroup_info[ind+1] =="offline" and "leader" in vgroup_info: + tdLog.info(" === revote leader ok , leader is {} now ====".format(vgroup_info[list(vgroup_info).index("leader")-1])) + check_status = True + elif vgroup_info[ind+1] !="offline": + tdLog.info(" === dnode {} should be offline ".format(self.stop_dnode_id)) + else: + continue + break + return check_status + + def force_stop_dnode(self, dnode_id ): + + tdSql.query("show dnodes") + port = None + for dnode_info in tdSql.queryResult: + if dnode_id == dnode_info[0]: + port = dnode_info[1].split(":")[-1] + break + else: + continue + if port: + tdLog.info(" ==== dnode {} will be force stop by kill -9 ====".format(dnode_id)) + psCmd = '''netstat -anp|grep -w LISTEN|grep -w %s |grep -o "LISTEN.*"|awk '{print $2}'|cut -d/ -f1|head -n1''' %(port) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + ps_kill_taosd = ''' kill -9 {} '''.format(processID) + # print(ps_kill_taosd) + os.system(ps_kill_taosd) + + def sync_run_case(self): + # stop follower and insert datas , update tables and create new stables + tdDnodes=cluster.dnodes + for loop in range(self.loop_restart_times): + db_name = "sync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + self.stop_dnode_id = self._get_stop_dnode_id(db_name) + + # check rows of datas + + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # get leader info before stop + before_leader_infos = self.get_leader_infos(db_name) + + # begin stop dnode + # force stop taosd by kill -9 + self.force_stop_dnode(self.stop_dnode_id) + + self.wait_stop_dnode_OK() + + # vote leaders check + + # get leader info after stop + after_leader_infos = self.get_leader_infos(db_name) + + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + # append rows of stablename when dnode stop make sure revote leaders + + while not revote_status: + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + + if revote_status: + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + else: + tdLog.info("===== leader of database {} is not ok , append rows fail =====".format(db_name)) + + # begin start dnode + start = time.time() + tdDnodes[self.stop_dnode_id-1].starttaosd() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end -start) + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + def unsync_run_case(self): + + def _restart_dnode_of_db_unsync(dbname): + + tdDnodes=cluster.dnodes + self.stop_dnode_id = self._get_stop_dnode_id(dbname) + # begin restart dnode + # force stop taosd by kill -9 + # get leader info before stop + before_leader_infos = self.get_leader_infos(db_name) + self.force_stop_dnode(self.stop_dnode_id) + + self.wait_stop_dnode_OK() + + # check revote leader when restart servers + # get leader info after stop + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + # append rows of stablename when dnode stop make sure revote leaders + while not revote_status: + after_leader_infos = self.get_leader_infos(db_name) + revote_status = self.check_revote_leader_success(db_name ,before_leader_infos , after_leader_infos) + + tbname = "sub_{}_{}".format(stablename , 0) + tdLog.info(" ==== begin append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.append_rows_of_exists_tables(db_name ,stablename , tbname , 100 ) + tdLog.info(" ==== check append rows of exists table {} when dnode {} offline ====".format(tbname , self.stop_dnode_id)) + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=100) + + # create new stables + tdLog.info(" ==== create new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb1' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} offline ====".format('new_stb1' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb1' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + # create new stables again + tdLog.info(" ==== create new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.create_stable_insert_datas(dbname = db_name , stablename = 'new_stb2' , tb_nums= 10 ,row_nums= 10 ) + tdLog.info(" ==== check new stable {} when dnode {} restart ====".format('new_stb2' , self.stop_dnode_id)) + self.check_insert_rows(db_name ,'new_stb2' ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + + tdDnodes[self.stop_dnode_id-1].starttaosd() + start = time.time() + self.wait_start_dnode_OK() + end = time.time() + time_cost = int(end-start) + + if time_cost > self.max_restart_time: + tdLog.exit(" ==== restart dnode {} cost too much time , please check ====".format(self.stop_dnode_id)) + + + def _create_threading(dbname): + self.current_thread = threading.Thread(target=_restart_dnode_of_db_unsync, args=(dbname,)) + return self.current_thread + + + ''' + in this mode , it will be extra threading control start or stop dnode , insert will always going with not care follower online or alive + ''' + for loop in range(self.loop_restart_times): + db_name = "unsync_db_{}".format(loop) + stablename = 'stable_{}'.format(loop) + self.create_database(dbname = db_name ,replica_num= self.replica , vgroup_nums= 1) + self.create_stable_insert_datas(dbname = db_name , stablename = stablename , tb_nums= 10 ,row_nums= 10 ) + + tdLog.info(" ===== restart dnode of database {} in an unsync threading ===== ".format(db_name)) + + # create sync threading and start it + self.current_thread = _create_threading(db_name) + self.current_thread.start() + + # check rows of datas + self.check_insert_rows(db_name ,stablename ,tb_nums=10 , row_nums= 10 ,append_rows=0) + + + self.current_thread.join() + + + def run(self): + + # basic insert and check of cluster + self.check_setup_cluster_status() + self.create_db_check_vgroups() + # self.sync_run_case() + self.unsync_run_case() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 974dcef723b7231318ff34357d6e35de166d038e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:18:42 +0800 Subject: [PATCH 11/36] other: merge 3.0 --- source/libs/nodes/src/nodesCloneFuncs.c | 3 + source/libs/nodes/src/nodesCodeFuncs.c | 21 ++ source/libs/planner/inc/planInt.h | 1 + source/libs/planner/src/planLogicCreater.c | 62 +++++- source/libs/planner/src/planOptimizer.c | 78 ++++++-- source/libs/planner/src/planPhysiCreater.c | 13 ++ source/libs/planner/src/planSpliter.c | 3 + source/libs/planner/src/planUtil.c | 183 ++++++++++++++++++ source/libs/planner/test/planIntervalTest.cpp | 6 + source/libs/planner/test/planSubqueryTest.cpp | 20 +- source/libs/stream/src/streamData.c | 2 + source/libs/stream/src/streamDispatch.c | 2 + source/libs/stream/src/streamExec.c | 1 + source/libs/sync/src/syncMain.c | 65 +++---- source/libs/sync/src/syncRaftCfg.c | 10 +- source/libs/transport/inc/transComm.h | 12 +- source/libs/transport/src/transCli.c | 57 +++--- source/libs/transport/src/transComm.c | 43 ++-- source/libs/transport/src/transSvr.c | 6 +- 19 files changed, 484 insertions(+), 104 deletions(-) diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index e7109b5a87..121e697630 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -332,6 +332,9 @@ static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) { COPY_SCALAR_FIELD(precision); CLONE_NODE_FIELD(pLimit); CLONE_NODE_FIELD(pSlimit); + COPY_SCALAR_FIELD(requireDataOrder); + COPY_SCALAR_FIELD(resultDataOrder); + COPY_SCALAR_FIELD(groupAction); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index e3c50de654..c92e6908f1 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -504,6 +504,9 @@ static const char* jkLogicPlanConditions = "Conditions"; static const char* jkLogicPlanChildren = "Children"; static const char* jkLogicPlanLimit = "Limit"; static const char* jkLogicPlanSlimit = "SLimit"; +static const char* jkLogicPlanRequireDataOrder = "RequireDataOrder"; +static const char* jkLogicPlanResultDataOrder = "ResultDataOrder"; +static const char* jkLogicPlanGroupAction = "GroupAction"; static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { const SLogicNode* pNode = (const SLogicNode*)pObj; @@ -521,6 +524,15 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkLogicPlanSlimit, nodeToJson, pNode->pSlimit); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicPlanRequireDataOrder, pNode->requireDataOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicPlanResultDataOrder, pNode->resultDataOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkLogicPlanGroupAction, pNode->groupAction); + } return code; } @@ -541,6 +553,15 @@ static int32_t jsonToLogicPlanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkLogicPlanSlimit, &pNode->pSlimit); } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkLogicPlanRequireDataOrder, pNode->requireDataOrder, code); + } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkLogicPlanResultDataOrder, pNode->resultDataOrder, code); + } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkLogicPlanGroupAction, pNode->groupAction, code); + } return code; } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index b7fba83235..88c7c26276 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -35,6 +35,7 @@ int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); int32_t createColumnByRewriteExprs(SNodeList* pExprs, SNodeList** pList); int32_t createColumnByRewriteExpr(SNode* pExpr, SNodeList** pList); int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* pNew); +int32_t adjustLogicNodeDataRequirement(SLogicNode* pNode, EDataOrderLevel requirement); int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan); int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 9ced5c1cb6..366f970710 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -250,6 +250,9 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect SScanLogicNode* pScan = NULL; int32_t code = makeScanLogicNode(pCxt, pRealTable, pSelect->hasRepeatScanFuncs, (SLogicNode**)&pScan); + pScan->node.groupAction = GROUP_ACTION_NONE; + pScan->node.resultDataOrder = DATA_ORDER_LEVEL_IN_BLOCK; + // set columns to scan if (TSDB_CODE_SUCCESS == code) { code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, COLLECT_COL_TYPE_COL, @@ -336,6 +339,9 @@ static int32_t createJoinLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pJoin->joinType = pJoinTable->joinType; pJoin->isSingleTableJoin = pJoinTable->table.singleTable; + pJoin->node.groupAction = GROUP_ACTION_CLEAR; + pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; + pJoin->node.requireDataOrder = DATA_ORDER_LEVEL_GLOBAL; int32_t code = TSDB_CODE_SUCCESS; @@ -472,6 +478,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } pAgg->hasLastRow = pSelect->hasLastRowFunc; + pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; + pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; @@ -540,6 +549,10 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt pIdfRowsFunc->isTailFunc = pSelect->hasTailFunc; pIdfRowsFunc->isUniqueFunc = pSelect->hasUniqueFunc; pIdfRowsFunc->isTimeLineFunc = pSelect->hasTimeLineFunc; + pIdfRowsFunc->node.groupAction = GROUP_ACTION_KEEP; + pIdfRowsFunc->node.requireDataOrder = + pIdfRowsFunc->isTimeLineFunc ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_NONE; + pIdfRowsFunc->node.resultDataOrder = pIdfRowsFunc->node.requireDataOrder; // indefinite rows functions and _select_values functions int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pIdfRowsFunc->pFuncs); @@ -571,6 +584,10 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return TSDB_CODE_OUT_OF_MEMORY; } + pInterpFunc->node.groupAction = GROUP_ACTION_KEEP; + pInterpFunc->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsInterpFunc, &pInterpFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); @@ -642,10 +659,12 @@ static int32_t createWindowLogicNodeByState(SLogicPlanContext* pCxt, SStateWindo } pWindow->winType = WINDOW_TYPE_STATE; + pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pWindow->pStateExpr = nodesCloneNode(pState->pExpr); - pWindow->pTspk = nodesCloneNode(pState->pCol); - if (NULL == pWindow->pTspk) { + if (NULL == pWindow->pStateExpr || NULL == pWindow->pTspk) { nodesDestroyNode((SNode*)pWindow); return TSDB_CODE_OUT_OF_MEMORY; } @@ -663,6 +682,9 @@ static int32_t createWindowLogicNodeBySession(SLogicPlanContext* pCxt, SSessionW pWindow->winType = WINDOW_TYPE_SESSION; pWindow->sessionGap = ((SValueNode*)pSession->pGap)->datum.i; pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? SESSION_ALGO_STREAM_SINGLE : SESSION_ALGO_MERGE; + pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.requireDataOrder = pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : DATA_ORDER_LEVEL_IN_GROUP; + pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pWindow->pTspk = nodesCloneNode((SNode*)pSession->pCol); if (NULL == pWindow->pTspk) { @@ -689,6 +711,9 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva pWindow->slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : pWindow->intervalUnit); pWindow->windowAlgo = pCxt->pPlanCxt->streamQuery ? INTERVAL_ALGO_STREAM_SINGLE : INTERVAL_ALGO_HASH; + pWindow->node.groupAction = GROUP_ACTION_KEEP; + pWindow->node.requireDataOrder = DATA_ORDER_LEVEL_IN_BLOCK; + pWindow->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; pWindow->pTspk = nodesCloneNode(pInterval->pCol); if (NULL == pWindow->pTspk) { @@ -734,6 +759,10 @@ static int32_t createFillLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return TSDB_CODE_OUT_OF_MEMORY; } + pFill->node.groupAction = GROUP_ACTION_KEEP; + pFill->node.requireDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + pFill->node.resultDataOrder = DATA_ORDER_LEVEL_IN_GROUP; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_WINDOW, NULL, COLLECT_COL_TYPE_ALL, &pFill->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pFill->node.pTargets) { code = nodesListMakeStrictAppend(&pFill->node.pTargets, @@ -768,6 +797,9 @@ static int32_t createSortLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect } pSort->groupSort = pSelect->groupSort; + pSort->node.groupAction = pSort->groupSort ? GROUP_ACTION_KEEP : GROUP_ACTION_CLEAR; + pSort->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; + pSort->node.resultDataOrder = pSort->groupSort ? DATA_ORDER_LEVEL_IN_GROUP : DATA_ORDER_LEVEL_GLOBAL; int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_ORDER_BY, NULL, COLLECT_COL_TYPE_ALL, &pSort->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pSort->node.pTargets) { @@ -818,6 +850,9 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel TSWAP(pProject->node.pLimit, pSelect->pLimit); TSWAP(pProject->node.pSlimit, pSelect->pSlimit); + pProject->node.groupAction = GROUP_ACTION_CLEAR; + pProject->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; + pProject->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; @@ -850,6 +885,10 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS return TSDB_CODE_OUT_OF_MEMORY; } + pPartition->node.groupAction = GROUP_ACTION_SET; + pPartition->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; + pPartition->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; + int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, NULL, COLLECT_COL_TYPE_ALL, &pPartition->node.pTargets); if (TSDB_CODE_SUCCESS == code && NULL == pPartition->node.pTargets) { @@ -882,11 +921,23 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return TSDB_CODE_OUT_OF_MEMORY; } + pAgg->node.groupAction = GROUP_ACTION_SET; + pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; + pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; + int32_t code = TSDB_CODE_SUCCESS; // set grouyp keys, agg funcs and having conditions - pAgg->pGroupKeys = nodesCloneList(pSelect->pProjectionList); - if (NULL == pAgg->pGroupKeys) { - code = TSDB_CODE_OUT_OF_MEMORY; + SNodeList* pGroupKeys = NULL; + SNode* pProjection = NULL; + FOREACH(pProjection, pSelect->pProjectionList) { + code = nodesListMakeStrictAppend(&pGroupKeys, createGroupingSetNode(pProjection)); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(pGroupKeys); + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + pAgg->pGroupKeys = pGroupKeys; } // rewrite the expression in subsequent clauses @@ -1361,6 +1412,7 @@ int32_t createLogicPlan(SPlanContext* pCxt, SLogicSubplan** pLogicSubplan) { if (TSDB_CODE_SUCCESS == code) { setLogicNodeParent(pSubplan->pNode); setLogicSubplanType(cxt.hasScan, pSubplan); + code = adjustLogicNodeDataRequirement(pSubplan->pNode, DATA_ORDER_LEVEL_NONE); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 36b58afb76..b006ac2b0a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1579,6 +1579,34 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return eliminateProjOptCheckProjColumnNames(pProjectNode); } +typedef struct CheckNewChildTargetsCxt { + SNodeList* pNewChildTargets; + bool canUse; +} CheckNewChildTargetsCxt; + +static EDealRes eliminateProjOptCanUseNewChildTargetsImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + CheckNewChildTargetsCxt* pCxt = pContext; + SNode* pTarget = NULL; + FOREACH(pTarget, pCxt->pNewChildTargets) { + if (!nodesEqualNode(pTarget, pNode)) { + pCxt->canUse = false; + return DEAL_RES_END; + } + } + } + return DEAL_RES_CONTINUE; +} + +static bool eliminateProjOptCanUseNewChildTargets(SLogicNode* pChild, SNodeList* pNewChildTargets) { + if (NULL == pChild->pConditions) { + return true; + } + CheckNewChildTargetsCxt cxt = {.pNewChildTargets = pNewChildTargets, .canUse = true}; + nodesWalkExpr(pChild->pConditions, eliminateProjOptCanUseNewChildTargetsImpl, &cxt); + return cxt.canUse; +} + static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SProjectLogicNode* pProjectNode) { SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0); @@ -1594,8 +1622,13 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } } } - nodesDestroyList(pChild->pTargets); - pChild->pTargets = pNewChildTargets; + if (eliminateProjOptCanUseNewChildTargets(pChild, pNewChildTargets)) { + nodesDestroyList(pChild->pTargets); + pChild->pTargets = pNewChildTargets; + } else { + nodesDestroyList(pNewChildTargets); + return TSDB_CODE_SUCCESS; + } int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild); if (TSDB_CODE_SUCCESS == code) { @@ -1873,6 +1906,8 @@ static int32_t rewriteUniqueOptCreateAgg(SIndefRowsFuncLogicNode* pIndef, SLogic TSWAP(pAgg->node.pChildren, pIndef->node.pChildren); optResetParent((SLogicNode*)pAgg); pAgg->node.precision = pIndef->node.precision; + pAgg->node.requireDataOrder = DATA_ORDER_LEVEL_IN_BLOCK; // first function requirement + pAgg->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; bool hasSelectPrimaryKey = false; @@ -1945,6 +1980,8 @@ static int32_t rewriteUniqueOptCreateProject(SIndefRowsFuncLogicNode* pIndef, SL TSWAP(pProject->node.pTargets, pIndef->node.pTargets); pProject->node.precision = pIndef->node.precision; + pProject->node.requireDataOrder = DATA_ORDER_LEVEL_NONE; + pProject->node.resultDataOrder = DATA_ORDER_LEVEL_NONE; int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; @@ -1973,12 +2010,17 @@ static int32_t rewriteUniqueOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeAppend(&pProject->pChildren, (SNode*)pAgg); - pAgg->pParent = pProject; - pAgg = NULL; } if (TSDB_CODE_SUCCESS == code) { + pAgg->pParent = pProject; + pAgg = NULL; code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pIndef, pProject); } + if (TSDB_CODE_SUCCESS == code) { + code = adjustLogicNodeDataRequirement( + pProject, NULL == pProject->pParent ? DATA_ORDER_LEVEL_NONE : pProject->pParent->requireDataOrder); + pProject = NULL; + } if (TSDB_CODE_SUCCESS == code) { nodesDestroyNode((SNode*)pIndef); } else { @@ -2145,11 +2187,20 @@ static bool tagScanMayBeOptimized(SLogicNode* pNode) { } SAggLogicNode* pAgg = (SAggLogicNode*)(pNode->pParent); - if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || - planOptNodeListHasCol(pAgg->pGroupKeys) || !planOptNodeListHasTbname(pAgg->pGroupKeys)) { + if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || planOptNodeListHasCol(pAgg->pGroupKeys) || + !planOptNodeListHasTbname(pAgg->pGroupKeys)) { return false; } - + + SNode* pGroupKey = NULL; + FOREACH(pGroupKey, pAgg->pGroupKeys) { + SNode* pGroup = NULL; + FOREACH(pGroup, ((SGroupingSetNode*)pGroupKey)->pParameterList) { + if (QUERY_NODE_COLUMN != nodeType(pGroup)) { + return false; + } + } + } return true; } @@ -2162,11 +2213,12 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp pScanNode->scanType = SCAN_TYPE_TAG; SNode* pTarget = NULL; FOREACH(pTarget, pScanNode->node.pTargets) { - if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)(pTarget))->colId) { - ERASE_NODE(pScanNode->node.pTargets); - break; - } + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)(pTarget))->colId) { + ERASE_NODE(pScanNode->node.pTargets); + break; + } } + NODES_DESTORY_LIST(pScanNode->pScanCols); SLogicNode* pAgg = pScanNode->node.pParent; @@ -2176,8 +2228,8 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp SNode* pAggTarget = NULL; FOREACH(pAggTarget, pAgg->pTargets) { SNode* pScanTarget = NULL; - FOREACH(pScanTarget, pScanNode->node.pTargets) { - if (0 == strcmp( ((SColumnNode*)pAggTarget)->colName, ((SColumnNode*)pAggTarget)->colName )) { + FOREACH(pScanTarget, pScanNode->node.pTargets) { + if (0 == strcmp(((SColumnNode*)pAggTarget)->colName, ((SColumnNode*)pAggTarget)->colName)) { nodesListAppend(pScanTargets, nodesCloneNode(pScanTarget)); break; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index ee2457e400..0a1f8bbd0b 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -974,6 +974,17 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh return code; } +static bool projectCanMergeDataBlock(SProjectLogicNode* pProject) { + if (DATA_ORDER_LEVEL_NONE == pProject->node.resultDataOrder) { + return true; + } + if (1 != LIST_LENGTH(pProject->node.pChildren)) { + return false; + } + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProject->node.pChildren, 0); + return DATA_ORDER_LEVEL_GLOBAL == pChild->resultDataOrder ? true : false; +} + static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode, SPhysiNode** pPhyNode) { SProjectPhysiNode* pProject = @@ -982,6 +993,8 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild return TSDB_CODE_OUT_OF_MEMORY; } + pProject->mergeDataBlock = projectCanMergeDataBlock(pProjectLogicNode); + int32_t code = TSDB_CODE_SUCCESS; if (0 == LIST_LENGTH(pChildren)) { pProject->pProjections = nodesCloneList(pProjectLogicNode->pProjections); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 4cbbf12385..a7ccb72792 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -657,6 +657,9 @@ static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitIn return TSDB_CODE_SUCCESS; } + if (NULL != pInfo->pSplitNode->pParent && QUERY_NODE_LOGIC_PLAN_FILL == nodeType(pInfo->pSplitNode->pParent)) { + pInfo->pSplitNode = pInfo->pSplitNode->pParent; + } SExchangeLogicNode* pExchange = NULL; int32_t code = splCreateExchangeNode(pCxt, pInfo->pSplitNode, &pExchange); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 77e4e05530..d4bd7e5162 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "functionMgt.h" #include "planInt.h" static char* getUsageErrFormat(int32_t errCode) { @@ -121,3 +122,185 @@ int32_t replaceLogicNode(SLogicSubplan* pSubplan, SLogicNode* pOld, SLogicNode* } return TSDB_CODE_PLAN_INTERNAL_ERROR; } + +static int32_t adjustScanDataRequirement(SScanLogicNode* pScan, EDataOrderLevel requirement) { + if (SCAN_TYPE_TABLE != pScan->scanType || SCAN_TYPE_TABLE_MERGE != pScan->scanType) { + return TSDB_CODE_SUCCESS; + } + // The lowest sort level of scan output data is DATA_ORDER_LEVEL_IN_BLOCK + if (requirement < DATA_ORDER_LEVEL_IN_BLOCK) { + requirement = DATA_ORDER_LEVEL_IN_BLOCK; + } + if (DATA_ORDER_LEVEL_IN_BLOCK == requirement) { + pScan->scanType = SCAN_TYPE_TABLE; + } else if (TSDB_SUPER_TABLE == pScan->tableType) { + pScan->scanType = SCAN_TYPE_TABLE_MERGE; + } + pScan->node.resultDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustJoinDataRequirement(SJoinLogicNode* pJoin, EDataOrderLevel requirement) { + // The lowest sort level of join input and output data is DATA_ORDER_LEVEL_GLOBAL + return TSDB_CODE_SUCCESS; +} + +static bool isKeepOrderAggFunc(SNodeList* pFuncs) { + SNode* pFunc = NULL; + FOREACH(pFunc, pFuncs) { + if (!fmIsKeepOrderFunc(((SFunctionNode*)pFunc)->funcId)) { + return false; + } + } + return true; +} + +static int32_t adjustAggDataRequirement(SAggLogicNode* pAgg, EDataOrderLevel requirement) { + // The sort level of agg with group by output data can only be DATA_ORDER_LEVEL_NONE + if (requirement > DATA_ORDER_LEVEL_NONE && (NULL != pAgg->pGroupKeys || !isKeepOrderAggFunc(pAgg->pAggFuncs))) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + pAgg->node.resultDataOrder = requirement; + pAgg->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustProjectDataRequirement(SProjectLogicNode* pProject, EDataOrderLevel requirement) { + pProject->node.resultDataOrder = requirement; + pProject->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustIntervalDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { + // The lowest sort level of interval output data is DATA_ORDER_LEVEL_IN_GROUP + if (requirement < DATA_ORDER_LEVEL_IN_GROUP) { + requirement = DATA_ORDER_LEVEL_IN_GROUP; + } + // The sort level of interval input data is always DATA_ORDER_LEVEL_IN_BLOCK + pWindow->node.resultDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustSessionDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { + if (requirement <= pWindow->node.resultDataOrder) { + return TSDB_CODE_SUCCESS; + } + pWindow->node.resultDataOrder = requirement; + pWindow->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustStateDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { + if (requirement <= pWindow->node.resultDataOrder) { + return TSDB_CODE_SUCCESS; + } + pWindow->node.resultDataOrder = requirement; + pWindow->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { + switch (pWindow->winType) { + case WINDOW_TYPE_INTERVAL: + return adjustIntervalDataRequirement(pWindow, requirement); + case WINDOW_TYPE_SESSION: + return adjustSessionDataRequirement(pWindow, requirement); + case WINDOW_TYPE_STATE: + return adjustStateDataRequirement(pWindow, requirement); + default: + break; + } + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t adjustFillDataRequirement(SFillLogicNode* pFill, EDataOrderLevel requirement) { + if (requirement <= pFill->node.requireDataOrder) { + return TSDB_CODE_SUCCESS; + } + pFill->node.resultDataOrder = requirement; + pFill->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustSortDataRequirement(SSortLogicNode* pSort, EDataOrderLevel requirement) { + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustPartitionDataRequirement(SPartitionLogicNode* pPart, EDataOrderLevel requirement) { + if (DATA_ORDER_LEVEL_GLOBAL == requirement) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; + } + pPart->node.resultDataOrder = requirement; + pPart->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustIndefRowsDataRequirement(SIndefRowsFuncLogicNode* pIndef, EDataOrderLevel requirement) { + if (requirement <= pIndef->node.resultDataOrder) { + return TSDB_CODE_SUCCESS; + } + pIndef->node.resultDataOrder = requirement; + pIndef->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustInterpDataRequirement(SInterpFuncLogicNode* pInterp, EDataOrderLevel requirement) { + if (requirement <= pInterp->node.requireDataOrder) { + return TSDB_CODE_SUCCESS; + } + pInterp->node.resultDataOrder = requirement; + pInterp->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + +int32_t adjustLogicNodeDataRequirement(SLogicNode* pNode, EDataOrderLevel requirement) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pNode)) { + case QUERY_NODE_LOGIC_PLAN_SCAN: + code = adjustScanDataRequirement((SScanLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_JOIN: + code = adjustJoinDataRequirement((SJoinLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_AGG: + code = adjustAggDataRequirement((SAggLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_PROJECT: + code = adjustProjectDataRequirement((SProjectLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY: + case QUERY_NODE_LOGIC_PLAN_EXCHANGE: + case QUERY_NODE_LOGIC_PLAN_MERGE: + break; + case QUERY_NODE_LOGIC_PLAN_WINDOW: + code = adjustWindowDataRequirement((SWindowLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_FILL: + code = adjustFillDataRequirement((SFillLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_SORT: + code = adjustSortDataRequirement((SSortLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_PARTITION: + code = adjustPartitionDataRequirement((SPartitionLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: + code = adjustIndefRowsDataRequirement((SIndefRowsFuncLogicNode*)pNode, requirement); + break; + case QUERY_NODE_LOGIC_PLAN_INTERP_FUNC: + code = adjustInterpDataRequirement((SInterpFuncLogicNode*)pNode, requirement); + break; + default: + break; + } + if (TSDB_CODE_SUCCESS == code) { + SNode* pChild = NULL; + FOREACH(pChild, pNode->pChildren) { + code = adjustLogicNodeDataRequirement((SLogicNode*)pChild, pNode->requireDataOrder); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + } + return code; +} diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index 9f34ead6ff..30f7051722 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -38,9 +38,15 @@ TEST_F(PlanIntervalTest, fill) { run("SELECT COUNT(*) FROM t1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " "INTERVAL(10s) FILL(LINEAR)"); + run("SELECT COUNT(*) FROM st1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "INTERVAL(10s) FILL(LINEAR)"); + run("SELECT COUNT(*), SUM(c1) FROM t1 " "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " "INTERVAL(10s) FILL(VALUE, 10, 20)"); + + run("SELECT COUNT(*) FROM st1 WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' " + "PARTITION BY TBNAME interval(10s) fill(prev)"); } TEST_F(PlanIntervalTest, selectFunc) { diff --git a/source/libs/planner/test/planSubqueryTest.cpp b/source/libs/planner/test/planSubqueryTest.cpp index 2ba3794d5c..16dd91846c 100644 --- a/source/libs/planner/test/planSubqueryTest.cpp +++ b/source/libs/planner/test/planSubqueryTest.cpp @@ -48,10 +48,28 @@ TEST_F(PlanSubqeuryTest, doubleGroupBy) { "WHERE a > 100 GROUP BY b"); } -TEST_F(PlanSubqeuryTest, withSetOperator) { +TEST_F(PlanSubqeuryTest, innerSetOperator) { useDb("root", "test"); run("SELECT c1 FROM (SELECT c1 FROM t1 UNION ALL SELECT c1 FROM t1)"); run("SELECT c1 FROM (SELECT c1 FROM t1 UNION SELECT c1 FROM t1)"); } + +TEST_F(PlanSubqeuryTest, innerFill) { + useDb("root", "test"); + + run("SELECT cnt FROM (SELECT _WSTART ts, COUNT(*) cnt FROM t1 " + "WHERE ts > '2022-04-01 00:00:00' and ts < '2022-04-30 23:59:59' INTERVAL(10s) FILL(LINEAR)) " + "WHERE ts > '2022-04-06 00:00:00'"); +} + +TEST_F(PlanSubqeuryTest, outerInterval) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM (SELECT * FROM st1) INTERVAL(5s)"); + + run("SELECT COUNT(*) + SUM(c1) FROM (SELECT * FROM st1) INTERVAL(5s)"); + + run("SELECT COUNT(*) FROM (SELECT ts, TOP(c1, 10) FROM st1s1) INTERVAL(5s)"); +} diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index d476980393..eb14990c0e 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -34,6 +34,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); + pDataBlock->info.version = be64toh(pRetrieve->version); pDataBlock->info.type = pRetrieve->streamBlockType; pDataBlock->info.childId = pReq->upstreamChildId; @@ -54,6 +55,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); + pDataBlock->info.version = be64toh(pRetrieve->version); pDataBlock->info.type = pRetrieve->streamBlockType; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5dec33d0fb..5d4adb2896 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -108,6 +108,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->numOfCols = htonl(numOfCols); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); + pRetrieve->version = htobe64(pBlock->info.version); int32_t actualLen = 0; blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); @@ -182,6 +183,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->numOfRows = htonl(pBlock->info.rows); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); + pRetrieve->version = htobe64(pBlock->info.version); int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index a8192b49f3..52b610228e 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -159,6 +159,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { if (data == NULL) { data = qItem; streamQueueProcessSuccess(pTask->inputQueue); + if (pTask->execType == TASK_EXEC__NONE) break; /*if (qItem->type == STREAM_INPUT__DATA_BLOCK) {*/ /*streamUpdateVer(pTask, (SStreamDataBlock*)qItem);*/ /*}*/ diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index e0133641b3..a453b2572c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1550,12 +1550,12 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { char logBuf[256 + 256]; if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 - ", snapshot:%" PRId64 ", snapshot-term:%" PRIu64 - ", standby:%d, " - "strategy:%d, batch:%d, " - "replica-num:%d, " - "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", + "vgId:%d, sync %s %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", snap-tm:%" PRIu64 + ", sby:%d, " + "stgy:%d, bch:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, @@ -1573,12 +1573,12 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { char* s = (char*)taosMemoryMalloc(len); if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 - ", snapshot:%" PRId64 ", snapshot-term:%" PRIu64 - ", standby:%d, " - "strategy:%d, batch:%d, " - "replica-num:%d, " - "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", + "vgId:%d, sync %s %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", snap-tm:%" PRIu64 + ", sby:%d, " + "stgy:%d, bch:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, @@ -1621,12 +1621,12 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { char logBuf[256 + 256]; if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(logBuf, sizeof(logBuf), - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 - ", snapshot:%" PRId64 ", snapshot-term:%" PRIu64 - ", standby:%d, " - "strategy:%d, batch:%d, " - "replica-num:%d, " - "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", + "vgId:%d, sync %s %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", snap-tm:%" PRIu64 + ", sby:%d, " + "stgy:%d, bch:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, @@ -1642,12 +1642,12 @@ void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { char* s = (char*)taosMemoryMalloc(len); if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { snprintf(s, len, - "vgId:%d, sync %s %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 - ", snapshot:%" PRId64 ", snapshot-term:%" PRIu64 - ", standby:%d, " - "strategy:%d, batch:%d, " - "replica-num:%d, " - "lconfig:%" PRId64 ", changing:%d, restore:%d, %s", + "vgId:%d, sync %s %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", snap-tm:%" PRIu64 + ", sby:%d, " + "stgy:%d, bch:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d, %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, @@ -1675,11 +1675,10 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); snprintf(s, len, - "vgId:%d, sync %s, term:%" PRIu64 ", commit:%" PRId64 ", first:%" PRId64 ", last:%" PRId64 - ", snapshot:%" PRId64 - ", standby:%d, " - "replica-num:%d, " - "lconfig:%" PRId64 ", changing:%d, restore:%d", + "vgId:%d, sync %s, tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", snap:%" PRId64 + ", sby:%d, " + "r-num:%d, " + "lcfg:%" PRId64 ", chging:%d, rsto:%d", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish); @@ -2977,7 +2976,7 @@ void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", commit:%" PRId64 + ", pterm:%" PRIu64 ", cmt:%" PRId64 ", " "datalen:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, @@ -2992,7 +2991,7 @@ void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMs char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries from %s:%d {term:%" PRIu64 ", pre-index:%" PRIu64 ", pre-term:%" PRIu64 - ", commit:%" PRIu64 ", pterm:%" PRIu64 + ", cmt:%" PRIu64 ", pterm:%" PRIu64 ", " "datalen:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, @@ -3007,7 +3006,7 @@ void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", commit:%" PRId64 ", datalen:%d, count:%d}, %s", + ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount, s); syncNodeEventLog(pSyncNode, logBuf); @@ -3020,7 +3019,7 @@ void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntries char logBuf[256]; snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-batch from %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", commit:%" PRId64 ", datalen:%d, count:%d}, %s", + ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, pMsg->dataLen, pMsg->dataCount, s); syncNodeEventLog(pSyncNode, logBuf); diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 0bbeaaf5b0..c634a1bf49 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -109,10 +109,10 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { if (pSyncCfg != NULL) { int32_t len = 512; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); memset(s, 0, len); - snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); + snprintf(s, len, "{r-num:%d, my:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); char *p = s + strlen(s); for (int i = 0; i < pSyncCfg->replicaNum; ++i) { /* @@ -206,7 +206,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -285,7 +285,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); } - cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 9dd1a745d3..3fa6344009 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -229,8 +229,8 @@ typedef struct { int8_t stop; } SAsyncPool; -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); -void transDestroyAsyncPool(SAsyncPool* pool); +SAsyncPool* transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb); +void transAsyncPoolDestroy(SAsyncPool* pool); int transAsyncSend(SAsyncPool* pool, queue* mq); bool transAsyncPoolIsEmpty(SAsyncPool* pool); @@ -322,7 +322,7 @@ typedef struct STransReq { } STransReq; void transReqQueueInit(queue* q); -void* transReqQueuePushReq(queue* q); +void* transReqQueuePush(queue* q); void* transReqQueueRemove(void* arg); void transReqQueueClear(queue* q); @@ -393,9 +393,9 @@ typedef struct SDelayQueue { uv_loop_t* loop; } SDelayQueue; -int transDQCreate(uv_loop_t* loop, SDelayQueue** queue); -void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)); -int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); +int transDQCreate(uv_loop_t* loop, SDelayQueue** queue); +void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)); +SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); /* diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f94a7f3c37..9de8c273d9 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -26,7 +26,7 @@ typedef struct SCliConn { SConnBuffer readBuf; STransQueue cliMsgs; - queue conn; + queue q; uint64_t expireTime; STransCtx ctx; @@ -451,7 +451,7 @@ void cliTimeoutCb(uv_timer_t* handle) { while (p != NULL) { while (!QUEUE_IS_EMPTY(&p->conn)) { queue* h = QUEUE_HEAD(&p->conn); - SCliConn* c = QUEUE_DATA(h, SCliConn, conn); + SCliConn* c = QUEUE_DATA(h, SCliConn, q); if (c->expireTime < currentTime) { QUEUE_REMOVE(h); transUnrefCliHandle(c); @@ -475,7 +475,7 @@ void* destroyConnPool(void* pool) { while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conn)) { queue* h = QUEUE_HEAD(&connList->conn); - SCliConn* c = QUEUE_DATA(h, SCliConn, conn); + SCliConn* c = QUEUE_DATA(h, SCliConn, q); cliDestroyConn(c, true); } connList = taosHashIterate((SHashObj*)pool, connList); @@ -501,11 +501,11 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { return NULL; } queue* h = QUEUE_HEAD(&plist->conn); - SCliConn* conn = QUEUE_DATA(h, SCliConn, conn); + SCliConn* conn = QUEUE_DATA(h, SCliConn, q); conn->status = ConnNormal; - QUEUE_REMOVE(&conn->conn); - QUEUE_INIT(&conn->conn); - assert(h == &conn->conn); + QUEUE_REMOVE(&conn->q); + QUEUE_INIT(&conn->q); + assert(h == &conn->q); return conn; } static int32_t allocConnRef(SCliConn* conn, bool update) { @@ -560,8 +560,8 @@ static void addConnToPool(void* pool, SCliConn* conn) { SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before assert(plist != NULL); - QUEUE_INIT(&conn->conn); - QUEUE_PUSH(&plist->conn, &conn->conn); + QUEUE_INIT(&conn->q); + QUEUE_PUSH(&plist->conn, &conn->q); assert(!QUEUE_IS_EMPTY(&plist->conn)); } static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { @@ -614,7 +614,7 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { transReqQueueInit(&conn->wreqQueue); transQueueInit(&conn->cliMsgs, NULL); - QUEUE_INIT(&conn->conn); + QUEUE_INIT(&conn->q); conn->hostThrd = pThrd; conn->status = ConnNormal; conn->broken = 0; @@ -626,8 +626,8 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { } static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); - QUEUE_REMOVE(&conn->conn); - QUEUE_INIT(&conn->conn); + QUEUE_REMOVE(&conn->q); + QUEUE_INIT(&conn->q); transRemoveExHandle(transGetRefMgt(), conn->refId); conn->refId = -1; @@ -735,7 +735,7 @@ void cliSend(SCliConn* pConn) { CONN_SET_PERSIST_BY_APP(pConn); } - uv_write_t* req = transReqQueuePushReq(&pConn->wreqQueue); + uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); return; _RETURN: @@ -990,7 +990,7 @@ static SCliThrd* createThrdObj() { pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 5, pThrd, cliAsyncCb); + pThrd->asyncPool = transAsyncPoolCreate(pThrd->loop, 5, pThrd, cliAsyncCb); uv_timer_init(pThrd->loop, &pThrd->timer); pThrd->timer.data = pThrd; @@ -1009,7 +1009,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { CLI_RELEASE_UV(pThrd->loop); taosThreadMutexDestroy(&pThrd->msgMtx); TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SCliMsg, destroyCmsg); - transDestroyAsyncPool(pThrd->asyncPool); + transAsyncPoolDestroy(pThrd->asyncPool); transDQDestroy(pThrd->delayQueue, destroyCmsg); taosMemoryFree(pThrd->loop); @@ -1054,6 +1054,12 @@ static void doDelayTask(void* param) { cliHandleReq(pMsg, pThrd); } +static void doCloseIdleConn(void* param) { + STaskArg* arg = param; + SCliConn* conn = arg->param1; + SCliThrd* pThrd = arg->param2; +} + static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; @@ -1075,7 +1081,7 @@ void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { } } -bool cliTryToExtractEpSet(STransMsg* pResp, SEpSet* dst) { +bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { if ((pResp == NULL || pResp->info.hasEpSet == 0)) { return false; } @@ -1116,7 +1122,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { */ STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; - bool retry = (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) ? true : false; + + bool retry = (pTransInst->retry != NULL && pTransInst->retry(code, pResp->msgType - 1)) ? true : false; if (retry) { pMsg->sent = 0; pCtx->retryCnt += 1; @@ -1125,6 +1132,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCnt < pCtx->retryLimit) { transUnrefCliHandle(pConn); EPSET_FORWARD_INUSE(&pCtx->epSet); + transFreeMsg(pResp->pCont); cliSchedMsgToNextNode(pMsg, pThrd); return -1; } @@ -1148,7 +1156,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STraceId* trace = &pResp->info.traceId; - bool hasEpSet = cliTryToExtractEpSet(pResp, &pCtx->epSet); + bool hasEpSet = cliTryExtractEpSet(pResp, &pCtx->epSet); if (hasEpSet) { char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); @@ -1336,19 +1344,18 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - if (0 != transAsyncSend(pThrd->asyncPool, &cliMsg->q)) { - tsem_destroy(sem); - taosMemoryFree(sem); + int ret = transAsyncSend(pThrd->asyncPool, &cliMsg->q); + if (ret != 0) { destroyCmsg(cliMsg); - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return -1; + goto _RETURN; } tsem_wait(sem); + +_RETURN: tsem_destroy(sem); taosMemoryFree(sem); - transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); - return 0; + return ret; } /* * diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index c3cba3118c..34849df2b2 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -175,7 +175,7 @@ int transSetConnOption(uv_tcp_t* stream) { return ret; } -SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { +SAsyncPool* transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) { SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); pool->nAsync = sz; pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); @@ -194,7 +194,7 @@ SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, int sz, void* arg, AsyncCB cb) return pool; } -void transDestroyAsyncPool(SAsyncPool* pool) { +void transAsyncPoolDestroy(SAsyncPool* pool) { for (int i = 0; i < pool->nAsync; i++) { uv_async_t* async = &(pool->asyncs[i]); // uv_close((uv_handle_t*)async, NULL); @@ -205,6 +205,14 @@ void transDestroyAsyncPool(SAsyncPool* pool) { taosMemoryFree(pool->asyncs); taosMemoryFree(pool); } +bool transAsyncPoolIsEmpty(SAsyncPool* pool) { + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + SAsyncItem* item = async->data; + if (!QUEUE_IS_EMPTY(&item->qmsg)) return false; + } + return true; +} int transAsyncSend(SAsyncPool* pool, queue* q) { if (atomic_load_8(&pool->stop) == 1) { return -1; @@ -228,14 +236,6 @@ int transAsyncSend(SAsyncPool* pool, queue* q) { } return uv_async_send(async); } -bool transAsyncPoolIsEmpty(SAsyncPool* pool) { - for (int i = 0; i < pool->nAsync; i++) { - uv_async_t* async = &(pool->asyncs[i]); - SAsyncItem* item = async->data; - if (!QUEUE_IS_EMPTY(&item->qmsg)) return false; - } - return true; -} void transCtxInit(STransCtx* ctx) { // init transCtx @@ -308,7 +308,7 @@ void transReqQueueInit(queue* q) { // init req queue QUEUE_INIT(q); } -void* transReqQueuePushReq(queue* q) { +void* transReqQueuePush(queue* q) { uv_write_t* req = taosMemoryCalloc(1, sizeof(uv_write_t)); STransReq* wreq = taosMemoryCalloc(1, sizeof(STransReq)); wreq->data = req; @@ -488,8 +488,25 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { heapDestroy(queue->heap); taosMemoryFree(queue); } +void transDQCancel(SDelayQueue* queue, SDelayTask* task) { + uv_timer_stop(queue->timer); -int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { + if (heapSize(queue->heap) <= 0) return; + heapRemove(queue->heap, &task->node); + + if (heapSize(queue->heap) != 0) { + HeapNode* minNode = heapMin(queue->heap); + if (minNode != NULL) return; + + uint64_t now = taosGetTimestampMs(); + SDelayTask* task = container_of(minNode, SDelayTask, node); + uint64_t timeout = now > task->execTime ? now - task->execTime : 0; + + uv_timer_start(queue->timer, transDQTimeout, timeout, 0); + } +} + +SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { uint64_t now = taosGetTimestampMs(); SDelayTask* task = taosMemoryCalloc(1, sizeof(SDelayTask)); task->func = func; @@ -507,7 +524,7 @@ int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_ tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->timer, timeoutMs); heapInsert(queue->heap, &task->node); uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0); - return 0; + return task; } void transPrintEpSet(SEpSet* pEpSet) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7b9402f954..3fb947bdba 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -434,7 +434,7 @@ static void uvStartSendRespInternal(SSvrMsg* smsg) { uvPrepareSendData(smsg, &wb); transRefSrvHandle(pConn); - uv_write_t* req = transReqQueuePushReq(&pConn->wreqQueue); + uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); uv_write(req, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); } static void uvStartSendResp(SSvrMsg* smsg) { @@ -697,7 +697,7 @@ static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { // conn set QUEUE_INIT(&pThrd->conn); - pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 1, pThrd, uvWorkerAsyncCb); + pThrd->asyncPool = transAsyncPoolCreate(pThrd->loop, 1, pThrd, uvWorkerAsyncCb); uv_pipe_connect(&pThrd->connect_req, pThrd->pipe, pipeName, uvOnPipeConnectionCb); // uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; @@ -976,7 +976,7 @@ void destroyWorkThrd(SWorkThrd* pThrd) { taosThreadJoin(pThrd->thread, NULL); SRV_RELEASE_UV(pThrd->loop); TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg); - transDestroyAsyncPool(pThrd->asyncPool); + transAsyncPoolDestroy(pThrd->asyncPool); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } From dc579e278e88824c1a4881195198f8d1070e5bf9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:19:20 +0800 Subject: [PATCH 12/36] other: merge 3.0 --- tests/pytest/crash_gen/crash_gen_main.py | 4 +- tests/pytest/util/common.py | 1 + tests/script/jenkins/basic.txt | 92 +- tests/script/tsim/insert/dupinsert.sim | 176 ---- tests/script/tsim/insert/update0.sim | 24 +- .../script/tsim/insert/update1_sort_merge.sim | 819 ++++++++++++++++++ tests/script/tsim/parser/auto_create_tb.sim | 2 +- tests/script/tsim/parser/join.sim | 15 +- tests/script/tsim/parser/join_manyblocks.sim | 2 - tests/script/tsim/parser/join_multivnode.sim | 59 +- tests/script/tsim/parser/last_groupby.sim | 12 +- tests/script/tsim/parser/lastrow_query.sim | 46 +- tests/script/tsim/parser/like.sim | 2 - tests/script/tsim/parser/limit1.sim | 5 +- tests/script/tsim/parser/limit1_stb.sim | 1 - tests/script/tsim/parser/limit1_tb.sim | 1 - .../script/tsim/parser/limit1_tblocks100.sim | 67 -- tests/script/tsim/parser/limit2.sim | 8 +- tests/script/tsim/parser/limit2_query.sim | 6 +- .../script/tsim/parser/limit2_tblocks100.sim | 76 -- tests/script/tsim/parser/limit_stb.sim | 1 - tests/script/tsim/parser/limit_tb.sim | 1 - tests/script/tsim/parser/line_insert.sim | 50 -- tests/script/tsim/parser/mixed_blocks.sim | 6 +- tests/script/tsim/parser/nchar.sim | 7 +- tests/script/tsim/parser/nestquery.sim | 95 +- tests/script/tsim/parser/null_char.sim | 17 +- tests/script/tsim/parser/precision_ns.sim | 4 +- .../tsim/parser/projection_limit_offset.sim | 41 +- tests/script/tsim/parser/regex.sim | 23 +- tests/script/tsim/parser/selectResNum.sim | 6 +- .../tsim/parser/select_across_vnodes.sim | 4 +- .../tsim/parser/select_distinct_tag.sim | 3 +- .../tsim/parser/select_from_cache_disk.sim | 6 +- tests/script/tsim/parser/select_with_tags.sim | 9 +- tests/script/tsim/parser/set_tag_vals.sim | 22 +- .../tsim/parser/single_row_in_tb_query.sim | 9 +- tests/script/tsim/parser/sliding.sim | 129 +-- tests/script/tsim/parser/union.sim | 148 +--- tests/script/tsim/parser/union_sysinfo.sim | 35 + tests/script/tsim/stream/basic1.sim | 16 +- tests/script/tsim/tag/commit.sim | 2 +- tests/script/tsim/valgrind/checkError5.sim | 10 + 43 files changed, 1132 insertions(+), 930 deletions(-) delete mode 100644 tests/script/tsim/insert/dupinsert.sim create mode 100644 tests/script/tsim/insert/update1_sort_merge.sim delete mode 100644 tests/script/tsim/parser/limit1_tblocks100.sim delete mode 100644 tests/script/tsim/parser/limit2_tblocks100.sim delete mode 100644 tests/script/tsim/parser/line_insert.sim create mode 100644 tests/script/tsim/parser/union_sysinfo.sim diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 0aea6e3e14..8990c24305 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -809,6 +809,8 @@ class StateEmpty(AnyState): ] def verifyTasksToState(self, tasks, newState): + if Config.getConfig().ignore_errors: # if we are asked to ignore certain errors, let's not verify CreateDB success. + return if (self.hasSuccess(tasks, TaskCreateDb) ): # at EMPTY, if there's succes in creating DB if (not self.hasTask(tasks, TaskDropDb)): # and no drop_db tasks @@ -2491,7 +2493,7 @@ class MainExec: action='store', default=None, type=str, - help='Ignore error codes, comma separated, 0x supported (default: None)') + help='Ignore error codes, comma separated, 0x supported, also suppresses certain transition state checks. (default: None)') parser.add_argument( '-i', '--num-replicas', diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index 7133e8365d..7a5b70d6ca 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -28,6 +28,7 @@ from util.common import * from util.constant import * from dataclasses import dataclass,field from typing import List +from datetime import datetime @dataclass class DataSet: diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index f319de4c2f..052bc62f59 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -27,11 +27,11 @@ ./test.sh -f tsim/db/delete_writing2.sim # unsupport ./test.sh -f tsim/db/dropdnodes.sim ./test.sh -f tsim/db/error1.sim -# jira ./test.sh -f tsim/db/keep.sim +# TD-17592 ./test.sh -f tsim/db/keep.sim ./test.sh -f tsim/db/len.sim ./test.sh -f tsim/db/repeat.sim ./test.sh -f tsim/db/show_create_db.sim -# jira ./test.sh -f tsim/db/show_create_table.sim +./test.sh -f tsim/db/show_create_table.sim ./test.sh -f tsim/db/tables.sim ./test.sh -f tsim/db/taosdlog.sim @@ -81,78 +81,73 @@ ./test.sh -f tsim/insert/query_multi_file.sim ./test.sh -f tsim/insert/tcp.sim ./test.sh -f tsim/insert/update0.sim +./test.sh -f tsim/insert/update1_sort_merge.sim # ---- parser ./test.sh -f tsim/parser/alter__for_community_version.sim ./test.sh -f tsim/parser/alter_column.sim ./test.sh -f tsim/parser/alter_stable.sim ./test.sh -f tsim/parser/alter.sim -# jira ./test.sh -f tsim/parser/alter1.sim +# ./test.sh -f tsim/parser/alter1.sim ./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim -# jira ./test.sh -f tsim/parser/auto_create_tb.sim +./test.sh -f tsim/parser/auto_create_tb.sim ./test.sh -f tsim/parser/between_and.sim ./test.sh -f tsim/parser/binary_escapeCharacter.sim -# jira ./test.sh -f tsim/parser/col_arithmetic_operation.sim -# jira ./test.sh -f tsim/parser/columnValue.sim +# ./test.sh -f tsim/parser/col_arithmetic_operation.sim +# ./test.sh -f tsim/parser/columnValue.sim ./test.sh -f tsim/parser/commit.sim -# jira ./test.sh -f tsim/parser/condition.sim +# TD-17661 ./test.sh -f tsim/parser/condition.sim ./test.sh -f tsim/parser/constCol.sim ./test.sh -f tsim/parser/create_db.sim ./test.sh -f tsim/parser/create_mt.sim -# jira ./test.sh -f tsim/parser/create_tb_with_tag_name.sim +# TD-17653 ./test.sh -f tsim/parser/create_tb_with_tag_name.sim ./test.sh -f tsim/parser/create_tb.sim ./test.sh -f tsim/parser/dbtbnameValidate.sim ./test.sh -f tsim/parser/distinct.sim -# jira ./test.sh -f tsim/parser/fill_stb.sim +# TD-17623 ./test.sh -f tsim/parser/fill_stb.sim ./test.sh -f tsim/parser/fill_us.sim ./test.sh -f tsim/parser/fill.sim ./test.sh -f tsim/parser/first_last.sim ./test.sh -f tsim/parser/fourArithmetic-basic.sim -# jira ./test.sh -f tsim/parser/function.sim +# TD-17659 TD-17658 ./test.sh -f tsim/parser/function.sim ./test.sh -f tsim/parser/groupby-basic.sim # ./test.sh -f tsim/parser/groupby.sim -# ./test.sh -f tsim/parser/having_child.sim -## ./test.sh -f tsim/parser/having.sim +# TD-17622 ./test.sh -f tsim/parser/having_child.sim +# ./test.sh -f tsim/parser/having.sim ./test.sh -f tsim/parser/import_commit1.sim ./test.sh -f tsim/parser/import_commit2.sim ./test.sh -f tsim/parser/import_commit3.sim -# jira ./test.sh -f tsim/parser/import_file.sim +# TD-17663 ./test.sh -f tsim/parser/import_file.sim ./test.sh -f tsim/parser/import.sim ./test.sh -f tsim/parser/insert_multiTbl.sim ./test.sh -f tsim/parser/insert_tb.sim -# jira ./test.sh -f tsim/parser/interp.sim -# ./test.sh -f tsim/parser/join.sim -# ./test.sh -f tsim/parser/join_manyblocks.sim -## ./test.sh -f tsim/parser/join_multitables.sim -# ./test.sh -f tsim/parser/join_multivnode.sim +# ./test.sh -f tsim/parser/interp.sim +./test.sh -f tsim/parser/join_manyblocks.sim +# ./test.sh -f tsim/parser/join_multitables.sim +# TD-17713 ./test.sh -f tsim/parser/join_multivnode.sim +# TD-17707 ./test.sh -f tsim/parser/join.sim ./test.sh -f tsim/parser/last_cache.sim -## ./test.sh -f tsim/parser/last_groupby.sim -# jira ./test.sh -f tsim/parser/lastrow.sim -## ./test.sh -f tsim/parser/like.sim +./test.sh -f tsim/parser/last_groupby.sim +# TD-17675 ./test.sh -f tsim/parser/lastrow.sim +./test.sh -f tsim/parser/like.sim # ./test.sh -f tsim/parser/limit.sim # ./test.sh -f tsim/parser/limit1.sim -# ./test.sh -f tsim/parser/limit1_tblocks100.sim -## ./test.sh -f tsim/parser/limit2.sim -## ./test.sh -f tsim/parser/limit2_tblocks100.sim -## ./test.sh -f tsim/parser/limit_stb.sim -## ./test.sh -f tsim/parser/limit_tb.sim -## ./test.sh -f tsim/parser/line_insert.sim -# ./test.sh -f tsim/parser/mixed_blocks.sim -# ./test.sh -f tsim/parser/nchar.sim -# ./test.sh -f tsim/parser/nestquery.sim +# ./test.sh -f tsim/parser/limit2.sim +./test.sh -f tsim/parser/mixed_blocks.sim +./test.sh -f tsim/parser/nchar.sim +# TD-17703 ./test.sh -f tsim/parser/nestquery.sim # ./test.sh -f tsim/parser/null_char.sim -## ./test.sh -f tsim/parser/precision_ns.sim -# ./test.sh -f tsim/parser/projection_limit_offset.sim -## ./test.sh -f tsim/parser/regex.sim -# ./test.sh -f tsim/parser/repeatAlter.sim -# ./test.sh -f tsim/parser/selectResNum.sim -# ./test.sh -f tsim/parser/select_across_vnodes.sim -# ./test.sh -f tsim/parser/select_distinct_tag.sim -# ./test.sh -f tsim/parser/select_from_cache_disk.sim +./test.sh -f tsim/parser/precision_ns.sim +./test.sh -f tsim/parser/projection_limit_offset.sim +./test.sh -f tsim/parser/regex.sim +./test.sh -f tsim/parser/select_across_vnodes.sim +./test.sh -f tsim/parser/select_distinct_tag.sim +./test.sh -f tsim/parser/select_from_cache_disk.sim # ./test.sh -f tsim/parser/select_with_tags.sim -# ./test.sh -f tsim/parser/set_tag_vals.sim -# ./test.sh -f tsim/parser/single_row_in_tb.sim -# ./test.sh -f tsim/parser/sliding.sim +./test.sh -f tsim/parser/selectResNum.sim +# TD-17685 ./test.sh -f tsim/parser/set_tag_vals.sim +./test.sh -f tsim/parser/single_row_in_tb.sim +# TD-17684 ./test.sh -f tsim/parser/sliding.sim # ./test.sh -f tsim/parser/slimit_alter_tags.sim # ./test.sh -f tsim/parser/slimit.sim # ./test.sh -f tsim/parser/slimit1.sim @@ -166,7 +161,8 @@ # ./test.sh -f tsim/parser/udf_dll_stable.sim # ./test.sh -f tsim/parser/udf_dll.sim # ./test.sh -f tsim/parser/udf.sim -# ./test.sh -f tsim/parser/union.sim +./test.sh -f tsim/parser/union.sim +# TD-17704 ./test.sh -f tsim/parser/union_sysinfo.sim # ./test.sh -f tsim/parser/where.sim # ---- query @@ -196,7 +192,7 @@ ./test.sh -f tsim/mnode/basic5.sim # ---- show -# jira ./test.sh -f tsim/show/basic.sim +./test.sh -f tsim/show/basic.sim # ---- table ./test.sh -f tsim/table/autocreate.sim @@ -328,7 +324,7 @@ ./test.sh -f tsim/vnode/stable_replica3_vnode3.sim # --- sync -# jira ./test.sh -f tsim/sync/3Replica1VgElect.sim +# ./test.sh -f tsim/sync/3Replica1VgElect.sim ./test.sh -f tsim/sync/3Replica5VgElect.sim ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim @@ -418,7 +414,7 @@ ./test.sh -f tsim/tag/3.sim ./test.sh -f tsim/tag/4.sim ./test.sh -f tsim/tag/5.sim -# jira ./test.sh -f tsim/tag/6.sim +# TD-17382 ./test.sh -f tsim/tag/6.sim ./test.sh -f tsim/tag/add.sim ./test.sh -f tsim/tag/bigint.sim ./test.sh -f tsim/tag/binary_binary.sim @@ -430,10 +426,10 @@ # ./test.sh -f tsim/tag/column.sim # ./test.sh -f tsim/tag/commit.sim # ./test.sh -f tsim/tag/create.sim -# ./test.sh -f tsim/tag/delete.sim -# jira ./test.sh -f tsim/tag/double.sim +# /test.sh -f tsim/tag/delete.sim +# ./test.sh -f tsim/tag/double.sim # ./test.sh -f tsim/tag/filter.sim -# jira ./test.sh -f tsim/tag/float.sim +# TD-17407 ./test.sh -f tsim/tag/float.sim ./test.sh -f tsim/tag/int_binary.sim ./test.sh -f tsim/tag/int_float.sim ./test.sh -f tsim/tag/int.sim diff --git a/tests/script/tsim/insert/dupinsert.sim b/tests/script/tsim/insert/dupinsert.sim deleted file mode 100644 index 0526cc19e4..0000000000 --- a/tests/script/tsim/insert/dupinsert.sim +++ /dev/null @@ -1,176 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -print =============== create database -sql drop database if exists d0 -sql create database d0 keep 365000d,365000d,365000d -sql use d0 - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20)); - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -sql create table ct1 using stb tags("BeiJing", "ChaoYang") -sql create table ct2 using stb tags("BeiJing", "HaiDian") -sql create table ct3 using stb tags("BeiJing", "PingGu") -sql create table ct4 using stb tags("BeiJing", "YanQing") - -sql show tables -if $rows != 4 then - print rows $rows != 4 - return -1 -endi - -print =============== step 1 insert records into ct1 - taosd merge -sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10, 20); -sql insert into ct1(ts,c1,c2,c3,c4) values('2022-05-03 16:59:00.011', 11, NULL, 'binary', 'nchar'); -sql insert into ct1 values('2022-05-03 16:59:00.016', 16, NULL, NULL, 'nchar', NULL); -sql insert into ct1 values('2022-05-03 16:59:00.016', 17, NULL, NULL, 'nchar', 170); -sql insert into ct1 values('2022-05-03 16:59:00.020', 20, NULL, NULL, 'nchar', 200); -sql insert into ct1 values('2022-05-03 16:59:00.016', 18, NULL, NULL, 'nchar', 180); -sql insert into ct1 values('2022-05-03 16:59:00.021', 21, NULL, NULL, 'nchar', 210); -sql insert into ct1 values('2022-05-03 16:59:00.022', 22, NULL, NULL, 'nchar', 220); - -print =============== step 2 insert records into ct1/ct2 - taosc merge for 2022-05-03 16:59:00.010 -sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10,10), ('2022-05-03 16:59:00.010',20,10.0), ('2022-05-03 16:59:00.010',30,NULL) ct2(ts,c1) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',20) ct1(ts,c2) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',100) ct1(ts,c3) values('2022-05-03 16:59:00.010','bin1'), ('2022-05-03 16:59:00.010','bin2') ct1(ts,c4,c5) values('2022-05-03 16:59:00.010',NULL,NULL), ('2022-05-03 16:59:00.010','nchar4',1000.01) ct2(ts,c2,c3,c4,c5) values('2022-05-03 16:59:00.010',20,'xkl','zxc',10); - -print =============== step 3 insert records into ct3 -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.020', 10,10); -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.021', 10,10), ('2022-05-03 16:59:00.021',20,20.0); -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.022', 30,30), ('2022-05-03 16:59:00.022',40,40.0),('2022-05-03 16:59:00.022',50,50.0); -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.023', 60,60), ('2022-05-03 16:59:00.023',70,70.0),('2022-05-03 16:59:00.023',80,80.0), ('2022-05-03 16:59:00.023',90,90.0); -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.024', 100,100), ('2022-05-03 16:59:00.025',110,110.0),('2022-05-03 16:59:00.025',120,120.0), ('2022-05-03 16:59:00.025',130,130.0); -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.030', 140,140), ('2022-05-03 16:59:00.030',150,150.0),('2022-05-03 16:59:00.031',160,160.0), ('2022-05-03 16:59:00.030',170,170.0), ('2022-05-03 16:59:00.031',180,180.0); -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.042', 190,190), ('2022-05-03 16:59:00.041',200,200.0),('2022-05-03 16:59:00.040',210,210.0); -sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.050', 220,220), ('2022-05-03 16:59:00.051',230,230.0),('2022-05-03 16:59:00.052',240,240.0); - -print =============== step 4 insert records into ct4 -sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.020', 10,'b0','n0'); -sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.021', 20,'b1','n1'), ('2022-05-03 16:59:00.021',30,'b2','n2'); -sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.022', 40,'b3','n3'), ('2022-05-03 16:59:00.022',40,'b4','n4'),('2022-05-03 16:59:00.022',50,'b5','n5'); -sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.023', 60,'b6','n6'), ('2022-05-03 16:59:00.024',70,'b7','n7'),('2022-05-03 16:59:00.024',80,'b8','n8'), ('2022-05-03 16:59:00.023',90,'b9','n9'); - - - -print =============== step 5 query records of ct1 from memory(taosc and taosd merge) -sql select * from ct1; -print $data00 $data01 $data02 $data03 $data04 $data05 -print $data10 $data11 $data12 $data13 $data14 $data15 -print $data20 $data21 $data22 $data23 $data24 $data25 -print $data30 $data31 $data32 $data33 $data34 $data35 -print $data40 $data41 $data42 $data43 $data44 $data45 -print $data50 $data51 $data52 $data53 $data54 $data55 - - - -print =============== step 6 query records of ct2 from memory(taosc and taosd merge) -sql select * from ct2; -print $data00 $data01 $data02 $data03 $data04 $data05 - -if $rows != 1 then - print rows $rows != 1 - return -1 -endi - - - -print =============== step 7 query records of ct3 from memory -sql select * from ct3; -print $data00 $data01 $data02 $data03 $data04 $data05 -print $data10 $data11 $data12 $data13 $data14 $data15 -print $data20 $data21 $data22 $data23 $data24 $data25 -print $data30 $data31 $data32 $data33 $data34 $data35 -print $data40 $data41 $data42 $data43 $data44 $data45 -print $data50 $data51 $data52 $data53 $data54 $data55 -print $data60 $data61 $data62 $data63 $data64 $data65 -print $data70 $data71 $data72 $data73 $data74 $data75 -print $data80 $data81 $data82 $data83 $data84 $data85 -print $data90 $data91 $data92 $data93 $data94 $data95 -print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] -print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] -print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] -print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] - -if $rows != 14 then - print rows $rows != 14 - return -1 -endi - - -print =============== step 8 query records of ct4 from memory -sql select * from ct4; -print $data00 $data01 $data02 $data03 $data04 $data05 -print $data10 $data11 $data12 $data13 $data14 $data15 -print $data20 $data21 $data22 $data23 $data24 $data25 -print $data30 $data31 $data32 $data33 $data34 $data35 -print $data40 $data41 $data42 $data43 $data44 $data45 - - -if $rows != 5 then - print rows $rows != 5 - return -1 -endi - - -#==================== reboot to trigger commit data to file -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode1 -s start - - -print =============== step 9 query records of ct1 from file -sql select * from ct1; -print $data00 $data01 $data02 $data03 $data04 $data05 -print $data10 $data11 $data12 $data13 $data14 $data15 -print $data20 $data21 $data22 $data23 $data24 $data25 -print $data30 $data31 $data32 $data33 $data34 $data35 -print $data40 $data41 $data42 $data43 $data44 $data45 -print $data50 $data51 $data52 $data53 $data54 $data55 - -if $rows != 6 then - print rows $rows != 6 - return -1 -endi - - -print =============== step 10 query records of ct2 from file -sql select * from ct2; -print $data00 $data01 $data02 $data03 $data04 $data05 - -if $rows != 1 then - print rows $rows != 1 - return -1 -endi - - -print =============== step 11 query records of ct3 from file -sql select * from ct3; -print $data00 $data01 $data02 $data03 $data04 $data05 -print $data10 $data11 $data12 $data13 $data14 $data15 -print $data20 $data21 $data22 $data23 $data24 $data25 -print $data30 $data31 $data32 $data33 $data34 $data35 -print $data40 $data41 $data42 $data43 $data44 $data45 -print $data50 $data51 $data52 $data53 $data54 $data55 -print $data60 $data61 $data62 $data63 $data64 $data65 -print $data70 $data71 $data72 $data73 $data74 $data75 -print $data80 $data81 $data82 $data83 $data84 $data85 -print $data90 $data91 $data92 $data93 $data94 $data95 -print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] -print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] -print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] -print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] - - -print =============== step 12 query records of ct4 from file -sql select * from ct4; -print $data00 $data01 $data02 $data03 $data04 $data05 -print $data10 $data11 $data12 $data13 $data14 $data15 -print $data20 $data21 $data22 $data23 $data24 $data25 -print $data30 $data31 $data32 $data33 $data34 $data35 -print $data40 $data41 $data42 $data43 $data44 $data45 \ No newline at end of file diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim index 09b8de08e6..c6843acb9d 100644 --- a/tests/script/tsim/insert/update0.sim +++ b/tests/script/tsim/insert/update0.sim @@ -79,8 +79,8 @@ if $rows != 3 then return -1 endi -if $data01 != 103 then - print data01 $data01 != 103 +if $data01 != 303 then + print data01 $data01 != 303 return -1 endi @@ -89,8 +89,8 @@ if $data11 != 80 then return -1 endi -if $data21 != 40 then - print data21 $data21 != 40 +if $data21 != 60 then + print data21 $data21 != 60 return -1 endi @@ -138,8 +138,8 @@ if $rows != 3 then return -1 endi -if $data01 != 103 then - print data01 $data01 != 103 +if $data01 != 303 then + print data01 $data01 != 303 return -1 endi @@ -148,8 +148,8 @@ if $data11 != 80 then return -1 endi -if $data21 != 40 then - print data21 $data21 != 40 +if $data21 != 60 then + print data21 $data21 != 60 return -1 endi @@ -208,8 +208,8 @@ if $data01 != 10 then return -1 endi -if $data11 != 103 then - print data11 $data11 != 103 +if $data11 != 303 then + print data11 $data11 != 303 return -1 endi @@ -218,8 +218,8 @@ if $data21 != NULL then return -1 endi -if $data31 != 40 then - print data31 $data31 != 40 +if $data31 != 60 then + print data31 $data31 != 60 return -1 endi diff --git a/tests/script/tsim/insert/update1_sort_merge.sim b/tests/script/tsim/insert/update1_sort_merge.sim new file mode 100644 index 0000000000..79d72b43a0 --- /dev/null +++ b/tests/script/tsim/insert/update1_sort_merge.sim @@ -0,0 +1,819 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== test case: merge duplicated rows in taosc and taosd +print =============== create database +sql drop database if exists d0 +sql create database d0 keep 365000d,365000d,365000d +sql use d0 + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20)); + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags("BeiJing", "ChaoYang") +sql create table ct2 using stb tags("BeiJing", "HaiDian") +sql create table ct3 using stb tags("BeiJing", "PingGu") +sql create table ct4 using stb tags("BeiJing", "YanQing") + +sql show tables +if $rows != 4 then + print rows $rows != 4 + return -1 +endi + +print =============== step 1 insert records into ct1 - taosd merge +sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10, 20); +sql insert into ct1(ts,c1,c2,c3,c4) values('2022-05-03 16:59:00.011', 11, NULL, 'binary', 'nchar'); +sql insert into ct1 values('2022-05-03 16:59:00.016', 16, NULL, NULL, 'nchar', NULL); +sql insert into ct1 values('2022-05-03 16:59:00.016', 17, NULL, NULL, 'nchar', 170); +sql insert into ct1 values('2022-05-03 16:59:00.020', 20, NULL, NULL, 'nchar', 200); +sql insert into ct1 values('2022-05-03 16:59:00.016', 18, NULL, NULL, 'nchar', 180); +sql insert into ct1 values('2022-05-03 16:59:00.021', 21, NULL, NULL, 'nchar', 210); +sql insert into ct1 values('2022-05-03 16:59:00.022', 22, NULL, NULL, 'nchar', 220); + +print =============== step 2 insert records into ct1/ct2 - taosc merge for 2022-05-03 16:59:00.010 +sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10,10), ('2022-05-03 16:59:00.010',20,10.0), ('2022-05-03 16:59:00.010',30,NULL) ct2(ts,c1) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',20) ct1(ts,c2) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',100) ct1(ts,c3) values('2022-05-03 16:59:00.010','bin1'), ('2022-05-03 16:59:00.010','bin2') ct1(ts,c4,c5) values('2022-05-03 16:59:00.010',NULL,NULL), ('2022-05-03 16:59:00.010','nchar4',1000.01) ct2(ts,c2,c3,c4,c5) values('2022-05-03 16:59:00.010',20,'xkl','zxc',10); + +print =============== step 3 insert records into ct3 +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.020', 10,10); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.021', 10,10), ('2022-05-03 16:59:00.021',20,20.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.022', 30,30), ('2022-05-03 16:59:00.022',40,40.0),('2022-05-03 16:59:00.022',50,50.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.023', 60,60), ('2022-05-03 16:59:00.023',70,70.0),('2022-05-03 16:59:00.023',80,80.0), ('2022-05-03 16:59:00.023',90,90.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.024', 100,100), ('2022-05-03 16:59:00.025',110,110.0),('2022-05-03 16:59:00.025',120,120.0), ('2022-05-03 16:59:00.025',130,130.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.030', 140,140), ('2022-05-03 16:59:00.030',150,150.0),('2022-05-03 16:59:00.031',160,160.0), ('2022-05-03 16:59:00.030',170,170.0), ('2022-05-03 16:59:00.031',180,180.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.042', 190,190), ('2022-05-03 16:59:00.041',200,200.0),('2022-05-03 16:59:00.040',210,210.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.050', 220,220), ('2022-05-03 16:59:00.051',230,230.0),('2022-05-03 16:59:00.052',240,240.0); + +print =============== step 4 insert records into ct4 +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.020', 10,'b0','n0'); +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.021', 20,'b1','n1'), ('2022-05-03 16:59:00.021',30,'b2','n2'); +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.022', 40,'b3','n3'), ('2022-05-03 16:59:00.022',40,'b4','n4'),('2022-05-03 16:59:00.022',50,'b5','n5'); +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.023', 60,'b6','n6'), ('2022-05-03 16:59:00.024',70,'b7','n7'),('2022-05-03 16:59:00.024',80,'b8','n8'), ('2022-05-03 16:59:00.023',90,'b9','n9'); + + + +print =============== step 5 query records of ct1 from memory(taosc and taosd merge) +sql select * from ct1; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 + +if $rows != 6 then + print rows $rows != 6 + return -1 +endi + +if $data01 != 30 then + print data01 $data01 != 30 + return -1 +endi + +if $data02 != 100.000000000 then + print data02 $data02 != 100.000000000 + return -1 +endi + +if $data03 != bin2 then + print data03 $data03 != bin2 + return -1 +endi + +if $data04 != nchar4 then + print data04 $data04 != nchar4 + return -1 +endi + +if $data05 != 1000.010000000 then + print data05 $data05 != 1000.010000000 + return -1 +endi + +if $data11 != 11 then + print data11 $data11 != 11 + return -1 +endi + +if $data12 != NULL then + print data12 $data12 != NULL + return -1 +endi + +if $data13 != binary then + print data13 $data13 != binary + return -1 +endi + +if $data14 != nchar then + print data14 $data14 != nchar + return -1 +endi + +if $data15 != NULL then + print data15 $data15 != NULL + return -1 +endi + +if $data51 != 22 then + print data51 $data51 != 22 + return -1 +endi + +if $data52 != NULL then + print data52 $data52 != NULL + return -1 +endi + +if $data53 != NULL then + print data53 $data53 != NULL + return -1 +endi + +if $data54 != nchar then + print data54 $data54 != nchar + return -1 +endi + +if $data55 != 220.000000000 then + print data55 $data55 != 220.000000000 + return -1 +endi + + +print =============== step 6 query records of ct2 from memory(taosc and taosd merge) +sql select * from ct2; +print $data00 $data01 $data02 $data03 $data04 $data05 + +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != 20 then + print data01 $data01 != 20 + return -1 +endi + +if $data02 != 20.000000000 then + print data02 $data02 != 20.000000000 + return -1 +endi + +if $data03 != xkl then + print data03 $data03 != xkl + return -1 +endi + +if $data04 != zxc then + print data04 $data04 != zxc + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +print =============== step 7 query records of ct3 from memory +sql select * from ct3; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 +print $data60 $data61 $data62 $data63 $data64 $data65 +print $data70 $data71 $data72 $data73 $data74 $data75 +print $data80 $data81 $data82 $data83 $data84 $data85 +print $data90 $data91 $data92 $data93 $data94 $data95 +print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] +print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] +print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] +print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] + +if $rows != 14 then + print rows $rows != 14 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 20 then + print data11 $data1 != 20 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 100 then + print data41 $data41 != 100 + return -1 +endi + +if $data51 != 130 then + print data51 $data51 != 130 + return -1 +endi + +if $data61 != 170 then + print data61 $data61 != 170 + return -1 +endi + +if $data71 != 180 then + print data71 $data71 != 180 + return -1 +endi + +if $data81 != 210 then + print data81 $data81 != 210 + return -1 +endi + +if $data91 != 200 then + print data91 $data91 != 200 + return -1 +endi + +if $data[10][1] != 190 then + print data[10][1] $data[10][1] != 190 + return -1 +endi + +if $data[11][1] != 220 then + print data[11][1] $data[11][1] != 220 + return -1 +endi + +if $data[12][1] != 230 then + print data[12][1] $data[12][1] != 230 + return -1 +endi + +if $data[13][1] != 240 then + print data[13][1] $data[13][1] != 240 + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +if $data15 != 20.000000000 then + print data15 $data5 != 20.000000000 + return -1 +endi + +if $data25 != 50.000000000 then + print data25 $data25 != 50.000000000 + return -1 +endi + +if $data35 != 90.000000000 then + print data35 $data35 != 90.000000000 + return -1 +endi + +if $data45 != 100.000000000 then + print data45 $data45 != 100.000000000 + return -1 +endi + +if $data55 != 130.000000000 then + print data55 $data55 != 130.000000000 + return -1 +endi + +if $data65 != 170.000000000 then + print data65 $data65 != 170.000000000 + return -1 +endi + +if $data75 != 180.000000000 then + print data75 $data75 != 180.000000000 + return -1 +endi + +if $data85 != 210.000000000 then + print data85 $data85 != 210.000000000 + return -1 +endi + +if $data95 != 200.000000000 then + print data95 $data95 != 200.000000000 + return -1 +endi + +if $data[10][5] != 190.000000000 then + print data[10][5] $data[10][5] != 190.000000000 + return -1 +endi + +if $data[11][5] != 220.000000000 then + print data[11][5] $data[11][5] != 220.000000000 + return -1 +endi + +if $data[12][5] != 230.000000000 then + print data[12][5] $data[12][5] != 230.000000000 + return -1 +endi + +if $data[13][5] != 240.000000000 then + print data[13][5] $data[13][5] != 240.000000000 + return -1 +endi + + +print =============== step 8 query records of ct4 from memory +sql select * from ct4; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 + + +if $rows != 5 then + print rows $rows != 5 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 30 then + print data11 $data11 != 30 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 80 then + print data41 $data41 != 80 + return -1 +endi + +if $data03 != b0 then + print data03 $data03 != b0 + return -1 +endi + +if $data13 != b2 then + print data13 $data13 != b2 + return -1 +endi + +if $data23 != b5 then + print data23 $data23 != b5 + return -1 +endi + +if $data33 != b9 then + print data33 $data33 != b9 + return -1 +endi + +if $data43 != b8 then + print data43 $data43 != b8 + return -1 +endi + +if $data04 != n0 then + print data04 $data04 != n0 + return -1 +endi + +if $data14 != n2 then + print data14 $data14 != n2 + return -1 +endi + +if $data24 != n5 then + print data24 $data24 != n5 + return -1 +endi + +if $data34 != n9 then + print data34 $data34 != n9 + return -1 +endi + +if $data44 != n8 then + print data44 $data44 != n8 + return -1 +endi + +#==================== reboot to trigger commit data to file +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + + + +print =============== step 9 query records of ct1 from file +sql select * from ct1; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 + +if $rows != 6 then + print rows $rows != 6 + return -1 +endi + +if $data01 != 30 then + print data01 $data01 != 30 + return -1 +endi + +if $data02 != 100.000000000 then + print data02 $data02 != 100.000000000 + return -1 +endi + +if $data03 != bin2 then + print data03 $data03 != bin2 + return -1 +endi + +if $data04 != nchar4 then + print data04 $data04 != nchar4 + return -1 +endi + +if $data05 != 1000.010000000 then + print data05 $data05 != 1000.010000000 + return -1 +endi + +if $data11 != 11 then + print data11 $data11 != 11 + return -1 +endi + +if $data12 != NULL then + print data12 $data12 != NULL + return -1 +endi + +if $data13 != binary then + print data13 $data13 != binary + return -1 +endi + +if $data14 != nchar then + print data14 $data14 != nchar + return -1 +endi + +if $data15 != NULL then + print data15 $data15 != NULL + return -1 +endi + +if $data51 != 22 then + print data51 $data51 != 22 + return -1 +endi + +if $data52 != NULL then + print data52 $data52 != NULL + return -1 +endi + +if $data53 != NULL then + print data53 $data53 != NULL + return -1 +endi + +if $data54 != nchar then + print data54 $data54 != nchar + return -1 +endi + +if $data55 != 220.000000000 then + print data55 $data55 != 220.000000000 + return -1 +endi + + +print =============== step 10 query records of ct2 from file +sql select * from ct2; +print $data00 $data01 $data02 $data03 $data04 $data05 + +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != 20 then + print data01 $data01 != 20 + return -1 +endi + +if $data02 != 20.000000000 then + print data02 $data02 != 20.000000000 + return -1 +endi + +if $data03 != xkl then + print data03 $data03 != xkl + return -1 +endi + +if $data04 != zxc then + print data04 $data04 != zxc + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +print =============== step 11 query records of ct3 from file +sql select * from ct3; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 +print $data60 $data61 $data62 $data63 $data64 $data65 +print $data70 $data71 $data72 $data73 $data74 $data75 +print $data80 $data81 $data82 $data83 $data84 $data85 +print $data90 $data91 $data92 $data93 $data94 $data95 +print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] +print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] +print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] +print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] + +if $rows != 14 then + print rows $rows != 14 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 20 then + print data11 $data1 != 20 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 100 then + print data41 $data41 != 100 + return -1 +endi + +if $data51 != 130 then + print data51 $data51 != 130 + return -1 +endi + +if $data61 != 170 then + print data61 $data61 != 170 + return -1 +endi + +if $data71 != 180 then + print data71 $data71 != 180 + return -1 +endi + +if $data81 != 210 then + print data81 $data81 != 210 + return -1 +endi + +if $data91 != 200 then + print data91 $data91 != 200 + return -1 +endi + +if $data[10][1] != 190 then + print data[10][1] $data[10][1] != 190 + return -1 +endi + +if $data[11][1] != 220 then + print data[11][1] $data[11][1] != 220 + return -1 +endi + +if $data[12][1] != 230 then + print data[12][1] $data[12][1] != 230 + return -1 +endi + +if $data[13][1] != 240 then + print data[13][1] $data[13][1] != 240 + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +if $data15 != 20.000000000 then + print data15 $data5 != 20.000000000 + return -1 +endi + +if $data25 != 50.000000000 then + print data25 $data25 != 50.000000000 + return -1 +endi + +if $data35 != 90.000000000 then + print data35 $data35 != 90.000000000 + return -1 +endi + +if $data45 != 100.000000000 then + print data45 $data45 != 100.000000000 + return -1 +endi + +if $data55 != 130.000000000 then + print data55 $data55 != 130.000000000 + return -1 +endi + +if $data65 != 170.000000000 then + print data65 $data65 != 170.000000000 + return -1 +endi + +if $data75 != 180.000000000 then + print data75 $data75 != 180.000000000 + return -1 +endi + +if $data85 != 210.000000000 then + print data85 $data85 != 210.000000000 + return -1 +endi + +if $data95 != 200.000000000 then + print data95 $data95 != 200.000000000 + return -1 +endi + +if $data[10][5] != 190.000000000 then + print data[10][5] $data[10][5] != 190.000000000 + return -1 +endi + +if $data[11][5] != 220.000000000 then + print data[11][5] $data[11][5] != 220.000000000 + return -1 +endi + +if $data[12][5] != 230.000000000 then + print data[12][5] $data[12][5] != 230.000000000 + return -1 +endi + +if $data[13][5] != 240.000000000 then + print data[13][5] $data[13][5] != 240.000000000 + return -1 +endi + + +print =============== step 12 query records of ct4 from file +sql select * from ct4; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 + + +if $rows != 5 then + print rows $rows != 5 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 30 then + print data11 $data11 != 30 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 80 then + print data41 $data41 != 80 + return -1 +endi + +if $data03 != b0 then + print data03 $data03 != b0 + return -1 +endi + +if $data13 != b2 then + print data13 $data13 != b2 + return -1 +endi + +if $data23 != b5 then + print data23 $data23 != b5 + return -1 +endi + +if $data33 != b9 then + print data33 $data33 != b9 + return -1 +endi + +if $data43 != b8 then + print data43 $data43 != b8 + return -1 +endi + +if $data04 != n0 then + print data04 $data04 != n0 + return -1 +endi + +if $data14 != n2 then + print data14 $data14 != n2 + return -1 +endi + +if $data24 != n5 then + print data24 $data24 != n5 + return -1 +endi + +if $data34 != n9 then + print data34 $data34 != n9 + return -1 +endi + +if $data44 != n8 then + print data44 $data44 != n8 + return -1 +endi \ No newline at end of file diff --git a/tests/script/tsim/parser/auto_create_tb.sim b/tests/script/tsim/parser/auto_create_tb.sim index 485f4f480c..3a64b79239 100644 --- a/tests/script/tsim/parser/auto_create_tb.sim +++ b/tests/script/tsim/parser/auto_create_tb.sim @@ -282,7 +282,7 @@ if $rows != 2 then return -1 endi -sql insert into tick_000001 ('ts', 'last_prc', 'volume', 'amount', 'oi', 'bid_prc1', 'ask_prc1') using tick tags (000001, Stocks) VALUES (1546391700000, 0.000000, 0, 0.000000, 0, 0.000000, 10.320000); +sql insert into tick_000001 (ts, last_prc, volume, amount, oi, bid_prc1, ask_prc1) using tick tags ('000001', 'Stocks') VALUES (1546391700000, 0.000000, 0, 0.000000, 0, 0.000000, 10.320000); sql select tbname from tick if $rows != 1 then return -1 diff --git a/tests/script/tsim/parser/join.sim b/tests/script/tsim/parser/join.sim index 55842d5c16..c052e24856 100644 --- a/tests/script/tsim/parser/join.sim +++ b/tests/script/tsim/parser/join.sim @@ -233,14 +233,23 @@ endi print 1 #select + where condition + interval query -sql select count(join_tb1.*) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts >= 100000 and join_tb0.c7 = true interval(10a) order by join_tb0.ts desc; - +print select count(join_tb1.*) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts >= 100000 and join_tb0.c7 = true interval(10a) order by _wstart asc; +sql select count(join_tb1.*) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts >= 100000 and join_tb0.c7 = true interval(10a) order by _wstart asc; $val = 100 if $rows != $val then return -1 endi -print 2 +print select count(join_tb1.*) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts >= 100000 and join_tb0.c7 = true interval(10a) order by _wstart desc; +sql select count(join_tb1.*) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts >= 100000 and join_tb0.c7 = true interval(10a) order by _wstart desc; +$val = 100 +if $rows != $val then + return -1 +endi + +#TODO +return + #===========================aggregation=================================== #select + where condition sql select count(join_tb1.*), count(join_tb0.*) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts >= 100000 and join_tb0.c7 = false; diff --git a/tests/script/tsim/parser/join_manyblocks.sim b/tests/script/tsim/parser/join_manyblocks.sim index eb5e34b079..154316a03f 100644 --- a/tests/script/tsim/parser/join_manyblocks.sim +++ b/tests/script/tsim/parser/join_manyblocks.sim @@ -73,8 +73,6 @@ while $i < $tbNum $tstart = 100000 endw -sleep 100 - print ===============join_manyblocks.sim print ==============> td-3313 sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; diff --git a/tests/script/tsim/parser/join_multivnode.sim b/tests/script/tsim/parser/join_multivnode.sim index a204b4fcea..c33fa85fa2 100644 --- a/tests/script/tsim/parser/join_multivnode.sim +++ b/tests/script/tsim/parser/join_multivnode.sim @@ -3,8 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -$dbPrefix = join_m_db -$tbPrefix = join_tb +$dbPrefix = join_db $mtPrefix = join_mt $tbNum = 3 $rowNum = 1000 @@ -14,6 +13,7 @@ print =============== join_multivnode.sim $i = 0 $db = $dbPrefix . $i $mt = $mtPrefix . $i +$tbPrefix = $mt . _tb $tstart = 100000 @@ -54,14 +54,12 @@ while $i < $tbNum $tstart = 100000 endw -sleep 100 - $tstart = 100000 -$mt = $mtPrefix . 1 . $i +$mt = $mtPrefix . 1 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12), t3 int) $i = 0 -$tbPrefix = join_1_tb +$tbPrefix = $mt . _tb while $i < $tbNum $tb = $tbPrefix . $i @@ -100,20 +98,19 @@ while $i < $tbNum endw print ===============multivnode projection join.sim - sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; - -print $row +print ===> rows $row if $row != 3000 then print expect 3000, actual: $row return -1 endi -print ======= second tags join +# TODO +return +print ======= second tags join sql create table m1(ts timestamp, k int) tags(a binary(12), b int); sql create table m2(ts timestamp, k int) tags(a binary(12), b int); - sql insert into tm1 using m1 tags('tm1', 1) values(10000000, 1) tm2 using m2 tags('tm2', 1) values(10000000, 99); sql select * from m1,m2 where m1.ts=m2.ts and m1.b=m2.b; @@ -122,9 +119,7 @@ if $row != 1 then endi sql select join_mt0.ts, join_mt1.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 - sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 - sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbname from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1 limit 1 #1970-01-01 08:01:40.800 | 10 | 45.000000000 | 0 | true | false | 0 | @@ -135,63 +130,49 @@ sql select count(join_mt0.c1), sum(join_mt0.c2)/count(*), avg(c2), first(join_mt if $rows != 300 then return -1 endi - if $data00 != @70-01-01 08:01:40.990@ then print expect 70-01-01 08:01:40.990, actual: $data00 return -1 endi - if $data01 != 10 then return -1 endi - if $data02 != 94.500000000 then print expect 94.500000000, actual $data02 return -1 endi - if $data03 != 94.500000000 then return -1 endi - if $data04 != 90 then return -1 endi - if $data05 != 1 then return -1 endi - if $data06 != 0 then return -1 endi - if $data10 != @70-01-01 08:01:40.980@ then print expect 70-01-01 08:01:40.980, actual: $data10 return -1 endi - if $data11 != 10 then return -1 endi - if $data12 != 84.500000000 then print expect 84.500000000, actual $data12 return -1 endi - if $data13 != 84.500000000 then return -1 endi - if $data14 != 80 then return -1 endi - if $data15 != 1 then return -1 endi - if $data16 != 0 then return -1 endi @@ -264,100 +245,76 @@ sql select m1.ts,m1.tbname,m1.a, m2.ts,m2.tbname,m2.b from m1,m2 where m1.a=m2.b if $rows != 4 then return -1 endi - if $data00 != @20-01-01 01:01:01.000@ then print expect 20-01-01 01:01:01.000, actual:$data00 return -1 endi - if $data01 != @tm0@ then return -1 endi - if $data02 != 0 then return -1 endi - if $data03 != @20-01-01 01:01:01.000@ then return -1 endi - if $data04 != @t0@ then return -1 endi - if $data05 != 0 then return -1 endi - if $data10 != @20-01-01 01:01:01.000@ then return -1 endi - if $data11 != @tm1@ then return -1 endi - if $data12 != 1 then return -1 endi - if $data13 != @20-01-01 01:01:01.000@ then return -1 endi - if $data14 != @t4@ then return -1 endi - if $data15 != 1 then return -1 endi - if $data20 != @20-01-01 01:01:01.000@ then return -1 endi - if $data21 != @tm4@ then return -1 endi - if $data22 != 4 then return -1 endi - if $data23 != @20-01-01 01:01:01.000@ then return -1 endi - if $data24 != @t1@ then return -1 endi - if $data25 != 4 then return -1 endi - if $data30 != @20-01-01 01:01:01.000@ then return -1 endi - if $data31 != @tm5@ then return -1 endi - if $data32 != 5 then return -1 endi - if $data33 != @20-01-01 01:01:01.000@ then return -1 endi - if $data34 != @t5@ then return -1 endi - if $data35 != 5 then return -1 endi diff --git a/tests/script/tsim/parser/last_groupby.sim b/tests/script/tsim/parser/last_groupby.sim index 8f9574412d..68d7f10fe2 100644 --- a/tests/script/tsim/parser/last_groupby.sim +++ b/tests/script/tsim/parser/last_groupby.sim @@ -4,14 +4,11 @@ system sh/exec.sh -n dnode1 -s start sql connect print ======================== dnode1 start - $db = testdb - sql create database $db sql use $db sql create stable st2 (ts timestamp, f1 int, f2 float, f3 double, f4 bigint, f5 smallint, f6 tinyint, f7 bool, f8 binary(10), f9 nchar(10)) tags (id1 int, id2 float, id3 nchar(10), id4 double, id5 smallint, id6 bigint, id7 binary(10)) - sql create table tb1 using st2 tags (1,1.0,"1",1.0,1,1,"1"); sql insert into tb1 values (now-200s,1,1.0,1.0,1,1,1,true,"1","1") @@ -23,16 +20,13 @@ sql insert into tb1 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") sql insert into tb1 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") sql insert into tb1 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") -sql select f1,last(*) from st2 group by f1; - +sql select f1, last(*) from st2 group by f1 order by f1; if $rows != 4 then return -1 endi - if $data00 != 1 then return -1 endi - if $data02 != 1 then print $data02 return -1 @@ -59,15 +53,13 @@ if $data09 != 1 then return -1 endi -sql select f1,last(f1,st2.*) from st2 group by f1; +sql select f1, last(f1,st2.*) from st2 group by f1 order by f1; if $rows != 4 then return -1 endi - if $data00 != 1 then return -1 endi - if $data01 != 1 then return -1 endi diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim index cb523d5c8e..5ffba38d14 100644 --- a/tests/script/tsim/parser/lastrow_query.sim +++ b/tests/script/tsim/parser/lastrow_query.sim @@ -66,32 +66,32 @@ if $row != 21600 then endi #regression test case 3 -sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 1 +sql select _wstart, t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 1 if $row != 2 then return -1 endi -#if $data01 != 7 then -# return -1 -#endi -#if $data02 != 7 then -# return -1 -#endi -#if $data03 != 59 then -# print expect 59, actual: $data03 -# return -1 -#endi -#if $data04 != 7 then -# return -1 -#endi -#if $data11 != 8 then -# return -1 -#endi -#if $data12 != 8 then -# return -1 -#endi -#if $data13 != NULL then -# return -1 -#endi +if $data01 != NULL then + return -1 +endi +if $data02 != NULL then + return -1 +endi +if $data03 != NULL then + return -1 +endi +if $data11 != 7 then + return -1 +endi +if $data12 != 7 then + return -1 +endi +if $data13 != 59 then + print expect 59, actual: $data03 + return -1 +endi +if $data14 != 7 then + return -1 +endi sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' partition by t1 interval(1h) fill(NULL) limit 9 if $rows != 18 then diff --git a/tests/script/tsim/parser/like.sim b/tests/script/tsim/parser/like.sim index e7c191ed92..96672aee3c 100644 --- a/tests/script/tsim/parser/like.sim +++ b/tests/script/tsim/parser/like.sim @@ -5,7 +5,6 @@ sql connect print ======================== dnode1 start - $db = testdb sql drop database if exists $db sql create database $db cachemodel 'last_value' @@ -32,7 +31,6 @@ if $rows != 2 then return -1 endi - sql select b from $table1 where b like 'table\_name' if $rows != 1 then return -1 diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim index 1f72999784..b6d0629c8f 100644 --- a/tests/script/tsim/parser/limit1.sim +++ b/tests/script/tsim/parser/limit1.sim @@ -18,7 +18,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db +sql create database $db cache 16 print ====== create tables sql use $db sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) @@ -57,11 +57,10 @@ run tsim/parser/limit1_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed run tsim/parser/limit1_tb.sim run tsim/parser/limit1_stb.sim -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/limit1_stb.sim b/tests/script/tsim/parser/limit1_stb.sim index 513e2fac02..879fd7882f 100644 --- a/tests/script/tsim/parser/limit1_stb.sim +++ b/tests/script/tsim/parser/limit1_stb.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = lm1_db diff --git a/tests/script/tsim/parser/limit1_tb.sim b/tests/script/tsim/parser/limit1_tb.sim index 300af7ac7b..5a7c1bc201 100644 --- a/tests/script/tsim/parser/limit1_tb.sim +++ b/tests/script/tsim/parser/limit1_tb.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = lm1_db diff --git a/tests/script/tsim/parser/limit1_tblocks100.sim b/tests/script/tsim/parser/limit1_tblocks100.sim deleted file mode 100644 index f541ea7158..0000000000 --- a/tests/script/tsim/parser/limit1_tblocks100.sim +++ /dev/null @@ -1,67 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -$dbPrefix = lm1_db -$tbPrefix = lm1_tb -$stbPrefix = lm1_stb -$tbNum = 10 -$rowNum = 10000 -$totalNum = $tbNum * $rowNum -$ts0 = 1537146000000 -$delta = 600000 -print ========== limit1.sim -$i = 0 -$db = $dbPrefix . $i -$stb = $stbPrefix . $i - -sql drop database $db -x step1 -step1: -sql create database $db cache 16 -print ====== create tables -sql use $db -sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) - -$i = 0 -$ts = $ts0 -$halfNum = $tbNum / 2 -while $i < $halfNum - $tbId = $i + $halfNum - $tb = $tbPrefix . $i - $tb1 = $tbPrefix . $tbId - sql create table $tb using $stb tags( $i ) - sql create table $tb1 using $stb tags( $tbId ) - - $x = 0 - while $x < $rowNum - $xs = $x * $delta - $ts = $ts0 + $xs - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - $binary = 'binary . $c - $binary = $binary . ' - $nchar = 'nchar . $c - $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) - $x = $x + 1 - endw - - $i = $i + 1 -endw -print ====== tables created - -run tsim/parser/limit1_tb.sim -run tsim/parser/limit1_stb.sim - -print ================== restart server to commit data into disk -system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 -system sh/exec.sh -n dnode1 -s start -print ================== server restart completed - -run tsim/parser/limit1_tb.sim -run tsim/parser/limit1_stb.sim - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/limit2.sim b/tests/script/tsim/parser/limit2.sim index af03c6bb7f..ca308fa6e7 100644 --- a/tests/script/tsim/parser/limit2.sim +++ b/tests/script/tsim/parser/limit2.sim @@ -1,10 +1,6 @@ system sh/stop_dnodes.sh - system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c rowsInFileBlock -v 255 system sh/exec.sh -n dnode1 -s start -sleep 100 sql connect $dbPrefix = lm2_db @@ -69,10 +65,8 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 +sleep 100 system sh/exec.sh -n dnode1 -s start print ================== server restart completed run tsim/parser/limit2_query.sim - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/parser/limit2_query.sim b/tests/script/tsim/parser/limit2_query.sim index c35fd369ca..8a2da7988d 100644 --- a/tests/script/tsim/parser/limit2_query.sim +++ b/tests/script/tsim/parser/limit2_query.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = lm2_db @@ -24,8 +23,11 @@ sql use $db ##### aggregation on stb with 6 tags + where + group by + limit offset $val1 = 1 $val2 = $tbNum - 1 -sql select count(*) from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0 +print select count(*) from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0 +sql select count(*), t1, t2, t3, t4, t5, t6 from $stb where t1 > $val1 and t1 < $val2 group by t1, t2, t3, t4, t5, t6 order by t1 asc limit 1 offset 0 $val = $tbNum - 3 + +print $rows $val if $rows != $val then return -1 endi diff --git a/tests/script/tsim/parser/limit2_tblocks100.sim b/tests/script/tsim/parser/limit2_tblocks100.sim deleted file mode 100644 index 0d87a41838..0000000000 --- a/tests/script/tsim/parser/limit2_tblocks100.sim +++ /dev/null @@ -1,76 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c rowsInFileBlock -v 255 -system sh/exec.sh -n dnode1 -s start -sleep 100 -sql connect - -$dbPrefix = lm2_db -$tbPrefix = lm2_tb -$stbPrefix = lm2_stb -$tbNum = 10 -$rowNum = 10000 -$totalNum = $tbNum * $rowNum -$ts0 = 1537146000000 -$delta = 600000 -$tsu = $rowNum * $delta -$tsu = $tsu - $delta -$tsu = $tsu + $ts0 - -print ========== limit2.sim -$i = 0 -$db = $dbPrefix . $i -$stb = $stbPrefix . $i - -sql drop database $db -x step1 -step1: -sql create database $db tblocks 100 -print ====== create tables -sql use $db -sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int, t2 nchar(20), t3 binary(20), t4 bigint, t5 smallint, t6 double) - -$i = 0 -$ts = $ts0 -$halfNum = $tbNum / 2 -while $i < $halfNum - $i1 = $i + $halfNum - $tb = $tbPrefix . $i - $tb1 = $tbPrefix . $i1 - $tgstr = 'tb . $i - $tgstr = $tgstr . ' - $tgstr1 = 'tb . $i1 - $tgstr1 = $tgstr1 . ' - sql create table $tb using $stb tags( $i , $tgstr , $tgstr , $i , $i , $i ) - sql create table $tb1 using $stb tags( $i1 , $tgstr1 , $tgstr1 , $i , $i , $i ) - - $x = 0 - while $x < $rowNum - $xs = $x * $delta - $ts = $ts0 + $xs - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - $binary = 'binary . $c - $binary = $binary . ' - $nchar = 'nchar . $c - $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) - sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) - $x = $x + 1 - endw - - $i = $i + 1 -endw -print ====== tables created - -#run tsim/parser/limit2_query.sim - -print ================== restart server to commit data into disk -system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 100 -system sh/exec.sh -n dnode1 -s start -print ================== server restart completed - -run tsim/parser/limit2_query.sim diff --git a/tests/script/tsim/parser/limit_stb.sim b/tests/script/tsim/parser/limit_stb.sim index ec7c0e0f13..a3064d59e9 100644 --- a/tests/script/tsim/parser/limit_stb.sim +++ b/tests/script/tsim/parser/limit_stb.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = lm_db diff --git a/tests/script/tsim/parser/limit_tb.sim b/tests/script/tsim/parser/limit_tb.sim index 4a93797d40..d0d14c5bfc 100644 --- a/tests/script/tsim/parser/limit_tb.sim +++ b/tests/script/tsim/parser/limit_tb.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = lm_db diff --git a/tests/script/tsim/parser/line_insert.sim b/tests/script/tsim/parser/line_insert.sim deleted file mode 100644 index cbd960bed6..0000000000 --- a/tests/script/tsim/parser/line_insert.sim +++ /dev/null @@ -1,50 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start -sql connect - -print =============== step1 -$db = testlp -$mte = ste -$mt = st -sql drop database $db -x step1 -step1: -sql create database $db precision 'us' -sql use $db -sql create stable $mte (ts timestamp, f int) TAGS(t1 bigint) - -line_insert st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000ns -line_insert st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64 1626006833640000000ns -line_insert ste,t2=5f64,t3=L"ste" c1=true,c2=4i64,c3="iam" 1626056811823316532ns -line_insert stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000ns -sql select * from st -if $rows != 2 then - return -1 -endi - -if $data00 != @21-07-11 20:33:53.639000@ then - return -1 -endi - -if $data02 != @passit@ then - return -1 -endi - -sql select * from stf -if $rows != 1 then - return -1 -endi - -sql select * from ste -if $rows != 1 then - return -1 -endi - -#print =============== clear -sql drop database $db -sql show databases -if $rows != 0 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/mixed_blocks.sim b/tests/script/tsim/parser/mixed_blocks.sim index 50229ab35a..76ac7c1c54 100644 --- a/tests/script/tsim/parser/mixed_blocks.sim +++ b/tests/script/tsim/parser/mixed_blocks.sim @@ -54,7 +54,6 @@ sql show databases print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed @@ -76,7 +75,7 @@ while $x < $rowNum endw #### query a STable and using where clause to filter out all the data from tb2 and make the query only return first/last of tb1 -sql select first(ts,c1), last(ts,c1), spread(c1) from $stb where c1 > 0 group by t1 +sql select first(ts,c1), last(ts,c1), spread(c1), t1 from $stb where c1 > 0 group by t1 if $rows != 1 then return -1 endi @@ -99,7 +98,7 @@ if $data05 != 1 then return -1 endi -sql select max(c1), min(c1), sum(c1), avg(c1), count(c1) from $stb where c1 > 0 group by t1 +sql select max(c1), min(c1), sum(c1), avg(c1), count(c1), t1 from $stb where c1 > 0 group by t1 if $rows != 1 then return -1 endi @@ -149,7 +148,6 @@ sql insert into t2 values('2020-1-1 1:5:1', 99); print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql select ts from m1 where ts='2020-1-1 1:5:1' diff --git a/tests/script/tsim/parser/nchar.sim b/tests/script/tsim/parser/nchar.sim index 52fc8b6864..ca94d964bb 100644 --- a/tests/script/tsim/parser/nchar.sim +++ b/tests/script/tsim/parser/nchar.sim @@ -228,7 +228,8 @@ sql_error select avg(tbcol) from $mt where tbcol1 = 1 group by tgcol sql_error select sum(tbcol) from $mt where tbcol1 = 1 group by tgcol sql_error select min(tbcol) from $mt where tbcol1 = 1 group by tgcol sql_error select max(tbcol) from $mt where tbcol1 = 1 group by tgcol -sql select first(tbcol) from $mt where tbcol1 = 1 group by tgcol + +sql select first(tbcol), tgcol from $mt where tbcol1 = 1 group by tgcol order by tgcol if $rows != 2 then return -1 endi @@ -244,7 +245,8 @@ endi if $data11 != 1 then return -1 endi -sql select last(tbcol) from $mt where tbcol1 = 1 group by tgcol + +sql select last(tbcol), tgcol from $mt where tbcol1 = 1 group by tgcol order by tgcol if $rows != 2 then return -1 endi @@ -265,6 +267,7 @@ sql create table stbb (ts timestamp, c1 nchar(5)) tags (t1 int) sql create table tbb1 using stbb tags(1) sql insert into tbb1 values ('2018-09-17 09:00:00', '涛思') sql insert into tbb1 values ('2018-09-17 09:00:01', 'insrt') + sql select * from tbb1 order by ts asc if $rows != 2 then return -1 diff --git a/tests/script/tsim/parser/nestquery.sim b/tests/script/tsim/parser/nestquery.sim index c82718c1cb..82d39eff8e 100644 --- a/tests/script/tsim/parser/nestquery.sim +++ b/tests/script/tsim/parser/nestquery.sim @@ -53,8 +53,6 @@ while $i < $half $i = $i + 1 endw -sleep 100 - $i = 1 $tb = $tbPrefix . $i @@ -63,7 +61,6 @@ sql select count(*) from (select count(*) from nest_mt0) if $rows != 1 then return -1 endi - if $data00 != 1 then return -1 endi @@ -72,35 +69,31 @@ sql select count(*) from (select count(*) from nest_mt0 group by tbname) if $rows != 1 then return -1 endi - if $data00 != 10 then return -1 endi -sql select count(*) from (select count(*) from nest_mt0 interval(10h) group by tbname) +sql select count(*) from (select count(*) from nest_mt0 partition by tbname interval(10h) ) if $rows != 1 then return -1 endi - if $data00 != 170 then return -1 endi -sql select sum(a) from (select count(*) a from nest_mt0 interval(10h) group by tbname) +sql select sum(a) from (select count(*) a from nest_mt0 partition by tbname interval(10h)) if $rows != 1 then return -1 endi - if $data00 != 100000 then return -1 endi print =================> alias name test -sql select ts from (select count(*) a from nest_tb0 interval(1h)) +sql select ts from (select _wstart as ts, count(*) a from nest_tb0 interval(1h)) if $rows != 167 then return -1 endi - if $data00 != @20-09-15 00:00:00.000@ then return -1 endi @@ -109,7 +102,6 @@ sql select count(a) from (select count(*) a from nest_tb0 interval(1h)) if $rows != 1 then return -1 endi - if $data00 != 167 then return -1 endi @@ -125,19 +117,16 @@ if $rows != 0 then return -1 endi -sql select * from (select count(*) a, tbname f1 from nest_mt0 group by tbname) t where t.a>0 and f1 = 'nest_tb0'; +sql select * from (select count(*) a, tbname f1, tbname from nest_mt0 group by tbname) t where t.a>0 and f1 = 'nest_tb0'; if $rows != 1 then return -1 endi - if $data00 != 10000 then return -1 endi - if $data01 != @nest_tb0@ then return -1 endi - if $data02 != @nest_tb0@ then return -1 endi @@ -145,37 +134,30 @@ endi print ===================> nest query interval sql_error select ts, avg(c1) from (select ts, c1 from nest_tb0); -sql select avg(c1) from (select * from nest_tb0) interval(3d) +sql select _wstart, avg(c1) from (select * from nest_tb0) interval(3d) if $rows != 3 then return -1 endi - if $data00 != @20-09-14 00:00:00.000@ then return -1 endi - if $data01 != 49.222222222 then return -1 endi - if $data10 != @20-09-17 00:00:00.000@ then - print expect 20-09-17 00:00:00.000, actual: $data10 return -1 endi - -if $data11 != 49.685185185 then +if $data11 != 49.581325301 then return -1 endi - if $data20 != @20-09-20 00:00:00.000@ then return -1 endi - -if $data21 != 49.500000000 then +if $data21 != 49.703539823 then return -1 endi -sql_error select stddev(c1) from (select c1 from nest_tb0); +sql select stddev(c1) from (select c1 from nest_tb0); sql_error select percentile(c1, 20) from (select * from nest_tb0); sql_error select interp(c1) from (select * from nest_tb0); sql_error select derivative(val, 1s, 0) from (select c1 val from nest_tb0); @@ -184,39 +166,31 @@ sql_error select irate(c1) from (select c1 from nest_tb0); sql_error select diff(c1), twa(c1) from (select * from nest_tb0); sql_error select irate(c1), interp(c1), twa(c1) from (select * from nest_tb0); -sql select apercentile(c1, 50) from (select * from nest_tb0) interval(1d) +sql select _wstart, apercentile(c1, 50) from (select * from nest_tb0) interval(1d) if $rows != 7 then return -1 endi - if $data00 != @20-09-15 00:00:00.000@ then return -1 endi - if $data01 != 47.571428571 then return -1 endi - if $data10 != @20-09-16 00:00:00.000@ then return -1 endi - if $data11 != 49.666666667 then return -1 endi - if $data20 != @20-09-17 00:00:00.000@ then return -1 endi - if $data21 != 49.000000000 then return -1 endi - if $data30 != @20-09-18 00:00:00.000@ then return -1 endi - if $data31 != 48.333333333 then return -1 endi @@ -225,7 +199,6 @@ sql select twa(c1) from (select * from nest_tb0); if $rows != 1 then return -1 endi - if $data00 != 49.500000000 then return -1 endi @@ -234,7 +207,6 @@ sql select leastsquares(c1, 1, 1) from (select * from nest_tb0); if $rows != 1 then return -1 endi - if $data00 != @{slop:0.000100, intercept:49.000000}@ then return -1 endi @@ -248,19 +220,15 @@ sql select derivative(c1, 1s, 0) from (select * from nest_tb0); if $rows != 9999 then return -1 endi - if $data00 != @20-09-15 00:01:00.000@ then return -1 endi - if $data01 != 0.016666667 then return -1 endi - if $data10 != @20-09-15 00:02:00.000@ then return -1 endi - if $data11 != 0.016666667 then return -1 endi @@ -274,54 +242,42 @@ sql select avg(c1),sum(c2), max(c3), min(c4), count(*), first(c7), last(c7),spre if $rows != 7 then return -1 endi - if $data00 != @20-09-15 00:00:00.000@ then return -1 endi - if $data01 != 48.666666667 then print expect 48.666666667, actual: $data01 return -1 endi - if $data02 != 70080.000000000 then return -1 endi - if $data03 != 99 then return -1 endi - if $data04 != 0 then return -1 endi - if $data05 != 1440 then return -1 endi - if $data06 != 0 then print $data06 return -1 endi - if $data07 != 1 then return -1 endi - if $data08 != 99.000000000 then print expect 99.000000000, actual: $data08 return -1 endi - if $data10 != @20-09-16 00:00:00.000@ then return -1 endi - if $data11 != 49.777777778 then return -1 endi - if $data12 != 71680.000000000 then return -1 endi @@ -332,39 +288,28 @@ sql select bottom(x, 20) from (select c1 x from nest_tb0) print ===================> group by + having - - print =========================> ascending order/descending order - - - print =========================> nest query join sql select a.ts,a.k,b.ts from (select count(*) k from nest_tb0 interval(30a)) a, (select count(*) f from nest_tb1 interval(30a)) b where a.ts = b.ts ; if $rows != 10000 then return -1 endi - if $data00 != @20-09-15 00:00:00.000@ then return -1 endi - if $data01 != 1 then return -1 endi - if $data02 != @20-09-15 00:00:00.000@ then return -1 endi - if $data10 != @20-09-15 00:01:00.000@ then return -1 endi - if $data11 != 1 then return -1 endi - if $data12 != @20-09-15 00:01:00.000@ then return -1 endi @@ -373,11 +318,9 @@ sql select sum(a.k), sum(b.f) from (select count(*) k from nest_tb0 interval(30a if $rows != 1 then return -1 endi - if $data00 != 10000 then return -1 endi - if $data01 != 10000 then return -1 endi @@ -386,19 +329,15 @@ sql select a.ts,a.k,b.ts,c.ts,c.ts,c.x from (select count(*) k from nest_tb0 int if $rows != 10000 then return -1 endi - if $data00 != @20-09-15 00:00:00.000@ then return -1 endi - if $data01 != 1 then return -1 endi - if $data02 != @20-09-15 00:00:00.000@ then return -1 endi - if $data03 != @20-09-15 00:00:00.000@ then return -1 endi @@ -407,11 +346,9 @@ sql select diff(val) from (select c1 val from nest_tb0); if $rows != 9999 then return -1 endi - if $data00 != @70-01-01 08:00:00.000@ then return -1 endi - if $data01 != 1 then return -1 endi @@ -425,19 +362,15 @@ sql select count(*),c1 from (select * from nest_tb0) where c1 < 2 group by c1; if $rows != 2 then return -1 endi - if $data00 != 100 then return -1 endi - if $data01 != 0 then return -1 endi - if $data10 != 100 then return -1 endi - if $data11 != 1 then return -1 endi @@ -447,11 +380,9 @@ sql select twa(c1) from nest_tb1 interval(19a); if $rows != 10000 then return -1 endi - if $data00 != @20-09-14 23:59:59.992@ then return -1 endi - if $data01 != 0.000083333 then return -1 endi @@ -461,19 +392,15 @@ sql select min(val),max(val),first(val),last(val),count(val),sum(val),avg(val) f if $rows != 1 then return -1 endi - if $data00 != 10000 then return -1 endi - if $data01 != 10000 then return -1 endi - if $data04 != 10 then return -1 endi - if $data05 != 100000 then return -1 endi @@ -487,19 +414,15 @@ sql select avg(k) from (select avg(k) k from t1 interval(1s)) interval(1m); if $rows != 2 then return -1 endi - if $data00 != @20-01-01 01:01:00.000000@ then return -1 endi - if $data01 != 1.000000000 then return -1 endi - if $data10 != @20-01-01 01:02:00.000000@ then return -1 endi - if $data11 != 2.000000000 then return -1 endi diff --git a/tests/script/tsim/parser/null_char.sim b/tests/script/tsim/parser/null_char.sim index 2bdb960968..fca4da78a0 100644 --- a/tests/script/tsim/parser/null_char.sim +++ b/tests/script/tsim/parser/null_char.sim @@ -78,6 +78,8 @@ endi #### case 1: tag NULL, or 'NULL' sql create table mt2 (ts timestamp, col1 int, col3 float, col5 binary(8), col6 bool, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag5 bool) sql create table st2 using mt2 tags (NULL, 'NULL', 102, 'true') +sql insert into st2 (ts, col1) values(now, 1) + sql select tag1, tag2, tag3, tag5 from st2 if $rows != 1 then return -1 @@ -115,6 +117,7 @@ if $rows != 0 then endi sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE') +sql insert into st3 (ts, col1) values(now, 1) sql select tag1, tag2, tag3, tag5 from st3 if $rows != 1 then return -1 @@ -134,9 +137,10 @@ if $data03 != 0 then endi ### bool: -sql_error create table stx using mt2 tags ('NULL', '123aBc', 104, '123') -sql_error create table sty using mt2 tags ('NULL', '123aBc', 104, 'xtz') +sql create table stx using mt2 tags ('NULL', '123aBc', 104, '123') +sql create table sty using mt2 tags ('NULL', '123aBc', 104, 'xtz') sql create table st4 using mt2 tags ('NULL', '123aBc', 104, 'NULL') +sql insert into st4 (ts, col1) values(now, 1) sql select tag1,tag2,tag3,tag5 from st4 if $rows != 1 then return -1 @@ -150,12 +154,13 @@ endi if $data02 != 104 then return -1 endi -if $data03 != NULL then +if $data03 != 0 then print ==6== expect: NULL, actually: $data03 return -1 endi sql create table st5 using mt2 tags ('NULL', '123aBc', 105, NULL) +sql insert into st5 (ts, col1) values(now, 1) sql select tag1,tag2,tag3,tag5 from st5 if $rows != 1 then return -1 @@ -173,8 +178,6 @@ if $data03 != NULL then return -1 endi - - #### case 2: dynamic create table using super table when insert into sql create table mt3 (ts timestamp, col1 int, col3 float, col5 binary(8), col6 bool, col9 nchar(8)) tags (tag1 binary(8), tag2 nchar(8), tag3 int, tag5 bool) sql_error insert into st31 using mt3 tags (NULL, 'NULL', 102, 'true') values (now+1s, 31, 31, 'bin_31', '123', 'nchar_31') @@ -182,10 +185,10 @@ sql_error insert into st32 using mt3 tags (NULL, 'ABC', 103, 'FALSE') values sql_error insert into st33 using mt3 tags ('NULL', '123aBc', 104, 'NULL') values (now+3s, 33, 33, 'bin_33', 'false123', 'nchar_33') sql_error insert into st34 using mt3 tags ('NULL', '123aBc', 105, NULL) values (now+4s, 34, 34.12345, 'bin_34', 'true123', 'nchar_34') - #### case 3: set tag value sql create table mt4 (ts timestamp, c1 int) tags (tag_binary binary(16), tag_nchar nchar(16), tag_int int, tag_bool bool, tag_float float, tag_double double) sql create table st41 using mt4 tags ("beijing", 'nchar_tag', 100, false, 9.12345, 7.123456789) +sql insert into st41 (ts, c1) values(now, 1) sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41 if $rows != 1 then return -1 @@ -245,7 +248,7 @@ endi ################### nchar sql alter table st41 set tag tag_nchar = "��˼����" sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41 -#sleep 100 + #if $data01 != ��˼���� then # print ==== expect ��˼����, actually $data01 # return -1 diff --git a/tests/script/tsim/parser/precision_ns.sim b/tests/script/tsim/parser/precision_ns.sim index bb822cd2b1..45b140f382 100644 --- a/tests/script/tsim/parser/precision_ns.sim +++ b/tests/script/tsim/parser/precision_ns.sim @@ -48,8 +48,6 @@ while $x < $rowNum $x = $x + 1 endw -sleep 100 - print =============== step2: select count(*) from tables $i = 0 $tb = $tbPrefix . $i @@ -103,7 +101,7 @@ sql select count(*) from $mt interval(100000000b) sliding(100000000b) print =============== clear sql drop database $db sql show databases -if $rows != 0 then +if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/parser/projection_limit_offset.sim b/tests/script/tsim/parser/projection_limit_offset.sim index 37f2e79995..5a96af2b3e 100644 --- a/tests/script/tsim/parser/projection_limit_offset.sim +++ b/tests/script/tsim/parser/projection_limit_offset.sim @@ -271,14 +271,14 @@ endi #[tbase-695] sql select ts,tbname from group_mt0 where ts>='1970-01-01 8:1:40' and ts<'1970-1-1 8:1:45' and c1<99999999 limit 100000 offset 5000 +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 +print ===> $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37 $data38 $data39 if $row != 35000 then return -1 endi -if $data00 != @70-01-01 08:01:40.000@ then - return -1 -endi - #=================================parse error sql========================================== sql_error select ts,tbname from group_mt0 order by ts desc limit 100 offset -1; sql_error select ts,tbname from group_mt0 order by c1 asc limit 100 offset -1; @@ -287,13 +287,13 @@ sql_error select ts,tbname from group_mt0 order by ts desc slimit -1, 100; sql_error select ts,tbname from group_mt0 order by ts desc slimit 1 soffset 1; #================================functions applys to sql=================================== -sql_error select first(t1) from group_mt0; -sql_error select last(t1) from group_mt0; -sql_error select min(t1) from group_mt0; -sql_error select max(t1) from group_mt0; -sql_error select top(t1, 20) from group_mt0; -sql_error select bottom(t1, 20) from group_mt0; -sql_error select avg(t1) from group_mt0; +sql select first(t1) from group_mt0; +sql select last(t1) from group_mt0; +sql select min(t1) from group_mt0; +sql select max(t1) from group_mt0; +sql select top(t1, 20) from group_mt0; +sql select bottom(t1, 20) from group_mt0; +sql select avg(t1) from group_mt0; sql_error select percentile(t1, 50) from group_mt0; sql_error select percentile(t1, 50) from group_mt0; sql_error select percentile(t1, 50) from group_mt0; @@ -309,7 +309,7 @@ endi #====================================tbase-716============================================== print tbase-716 -sql_error select count(*) from group_tb0 where ts in ('2016-1-1 12:12:12'); +sql select count(*) from group_tb0 where ts in ('2016-1-1 12:12:12'); sql_error select count(*) from group_tb0 where ts < '12:12:12'; #===============================sql for twa========================================== @@ -345,7 +345,7 @@ if $rows != 0 then return -1 endi -sql select count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by ts desc; +sql select _wstart, count(*),first(k),last(k) from m1 where tbname in ('tm0') interval(1s) order by _wstart desc; if $row != 5 then return -1 endi @@ -374,6 +374,7 @@ if $data13 != NULL then return -1 endi + print =============tbase-1324 sql select a, k-k from m1 if $row != 8 then @@ -384,8 +385,7 @@ sql select diff(k) from tm0 if $row != 3 then return -1 endi - -if $data21 != -1 then +if $data20 != -1 then return -1 endi @@ -395,20 +395,19 @@ sql_error select * from 1; sql_error select k+k; sql_error select k+1; sql_error select abc(); -sql_error select 1 where 1=2; -sql_error select 1 limit 1; -sql_error select 1 slimit 1; -sql_error select 1 interval(1h); +sql select 1 where 1=2; +sql select 1 limit 1; +sql select 1 slimit 1; +sql select 1 interval(1h); sql_error select count(*); sql_error select sum(k); -sql_error select 'abc'; +sql select 'abc'; sql_error select k+1,sum(k) from tm0; sql_error select k, sum(k) from tm0; sql_error select k, sum(k)+1 from tm0; print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/tsim/parser/regex.sim b/tests/script/tsim/parser/regex.sim index 41f52575d6..e5fb9f5f4e 100644 --- a/tests/script/tsim/parser/regex.sim +++ b/tests/script/tsim/parser/regex.sim @@ -18,12 +18,15 @@ sql create table $ct1_name using $st_name tags('taosdata1') sql create table $ct2_name using $st_name tags('taosdata2') sql create table not_match using $st_name tags('NOTMATCH') +sql insert into $ct1_name values(now, 'this is engine') +sql insert into $ct2_name values(now, 'this is app egnine') +sql insert into not_match values (now + 1s, '1234') + sql select tbname from $st_name where tbname match '.*' if $rows != 3 then return -1 endi - sql select tbname from $st_name where tbname match '^ct[[:digit:]]' if $rows != 2 then return -1 @@ -54,9 +57,6 @@ if $rows != 1 then return -1 endi -sql insert into $ct1_name values(now, 'this is engine') -sql insert into $ct2_name values(now, 'this is app egnine') - sql select c1b from $st_name where c1b match 'engine' if $data00 != @this is engine@ then return -1 @@ -66,12 +66,11 @@ if $rows != 1 then return -1 endi -sql select c1b from $st_name where c1b nmatch 'engine' +sql select c1b from $st_name where c1b nmatch 'engine' order by ts if $data00 != @this is app egnine@ then return -1 endi - -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -96,8 +95,8 @@ sql_error select * from wrong_type where c5 match '.*' sql_error select * from wrong_type where c5 nmatch '.*' sql_error select * from wrong_type where c6 match '.*' sql_error select * from wrong_type where c6 nmatch '.*' -sql_error select * from wrong_type where c7 match '.*' -sql_error select * from wrong_type where c7 nmatch '.*' +sql select * from wrong_type where c7 match '.*' +sql select * from wrong_type where c7 nmatch '.*' sql_error select * from wrong_type where t1 match '.*' sql_error select * from wrong_type where t1 nmatch '.*' sql_error select * from wrong_type where t2 match '.*' @@ -110,9 +109,7 @@ sql_error select * from wrong_type where t5 match '.*' sql_error select * from wrong_type where t5 nmatch '.*' sql_error select * from wrong_type where t6 match '.*' sql_error select * from wrong_type where t6 nmatch '.*' -sql_error select * from wrong_type where t7 match '.*' -sql_error select * from wrong_type where t7 nmatch '.*' +sql select * from wrong_type where t7 match '.*' +sql select * from wrong_type where t7 nmatch '.*' system sh/exec.sh -n dnode1 -s stop -x SIGINT - - diff --git a/tests/script/tsim/parser/selectResNum.sim b/tests/script/tsim/parser/selectResNum.sim index ac5ccd6e07..69ea2dccd5 100644 --- a/tests/script/tsim/parser/selectResNum.sim +++ b/tests/script/tsim/parser/selectResNum.sim @@ -20,7 +20,7 @@ $stb = $stbPrefix . $i sql drop database $db -x step1 step1: -sql create database $db cache 16 +sql create database $db print ====== create tables sql use $db sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) @@ -114,12 +114,8 @@ endw print ====== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ====== server restart completed -sleep 100 -sql connect -sleep 100 sql use $db ##### repeat test after server restart diff --git a/tests/script/tsim/parser/select_across_vnodes.sim b/tests/script/tsim/parser/select_across_vnodes.sim index 0ee011cf8a..c9696e563d 100644 --- a/tests/script/tsim/parser/select_across_vnodes.sim +++ b/tests/script/tsim/parser/select_across_vnodes.sim @@ -17,7 +17,7 @@ $db = $dbPrefix $stb = $stbPrefix sql drop database if exists $db -sql create database $db +sql create database $db vgroups 10 sql use $db print ====== create tables sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int) @@ -72,7 +72,7 @@ endi sql drop database $db sql show databases -if $rows != 0 then +if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/parser/select_distinct_tag.sim b/tests/script/tsim/parser/select_distinct_tag.sim index 92303ce64e..ec33ff8ac6 100644 --- a/tests/script/tsim/parser/select_distinct_tag.sim +++ b/tests/script/tsim/parser/select_distinct_tag.sim @@ -27,6 +27,7 @@ $ts = $ts0 while $i < $tbNum $tb = $tbPrefix . $i sql create table $tb using $stb tags( $i , 0 ) + sql insert into $tb (ts, c1) values (now, 1); $i = $i + 1 endw @@ -50,7 +51,7 @@ sql_error select distinct t1, t2 from &stb sql drop database $db sql show databases -if $rows != 0 then +if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/parser/select_from_cache_disk.sim b/tests/script/tsim/parser/select_from_cache_disk.sim index 2c9f359afe..0983e36a3a 100644 --- a/tests/script/tsim/parser/select_from_cache_disk.sim +++ b/tests/script/tsim/parser/select_from_cache_disk.sim @@ -31,17 +31,13 @@ sql insert into $tb values ('2018-09-17 09:00:00.030', 3) print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -sql connect -sleep 100 -sql use $db # generate some data in cache sql insert into $tb values ('2018-09-17 09:00:04.000', 4) sql insert into $tb values ('2018-09-17 09:00:04.010', 5) -sql select count(*) from $stb interval(1s) group by t1 +sql select _wstart, count(*), t1 from $stb partition by t1 interval(1s) order by _wstart if $rows != 2 then return -1 endi diff --git a/tests/script/tsim/parser/select_with_tags.sim b/tests/script/tsim/parser/select_with_tags.sim index b840a666f4..5130b39f48 100644 --- a/tests/script/tsim/parser/select_with_tags.sim +++ b/tests/script/tsim/parser/select_with_tags.sim @@ -62,9 +62,6 @@ while $i < $tbNum endw -sleep 100 - - #======================= only check first table tag, TD-4827 sql select count(*) from $mt where t1 in (0) if $rows != 1 then @@ -178,15 +175,15 @@ if $data03 != @abc15@ then endi sql select top(c6, 3) from select_tags_mt0 interval(10a) -sql select top(c3,10) from select_tags_mt0 interval(10a) group by tbname,t1,t2 -sql select top(c6, 3) from select_tags_mt0 interval(10a) group by tbname; +sql select top(c3,10) from select_tags_mt0 partition by tbname,t1,t2 interval(10a) +sql select top(c6, 3) from select_tags_mt0 partition by tbname interval(10a) sql select top(c6, 10) from select_tags_mt0 interval(10a); if $rows != 12800 then return -1 endi -sql select top(c1, 80), tbname, t1, t2 from select_tags_mt0; +sql select _rowts, top(c1, 80), tbname, t1, t2 from select_tags_mt0; if $rows != 80 then return -1 endi diff --git a/tests/script/tsim/parser/set_tag_vals.sim b/tests/script/tsim/parser/set_tag_vals.sim index 07b424ec6a..4bad716705 100644 --- a/tests/script/tsim/parser/set_tag_vals.sim +++ b/tests/script/tsim/parser/set_tag_vals.sim @@ -55,10 +55,8 @@ while $i < $tbNum $i = $i + 1 endw + print ====== tables created - -sleep 500 - sql show tables if $rows != $tbNum then return -1 @@ -74,12 +72,16 @@ while $i < $tbNum sql insert into $tb (ts, c1) values (now-100a, $i ) sql alter table $tb set tag t3 = $i sql insert into $tb (ts, c1) values (now, $i ) - sql alter table $tb set tag t4 = $i + + $name = ' . $i + $name = $name . ' + sql alter table $tb set tag t4 = $name $i = $i + 1 endw print ================== all tags have been changed! -sql_error select tbname from $stb where t3 = 'NULL' +sql reset query cache +sql select tbname from $stb where t3 = 'NULL' print ================== set tag to NULL sql create table stb1_tg (ts timestamp, c1 int) tags(t1 int,t2 bigint,t3 double,t4 float,t5 smallint,t6 tinyint) @@ -142,7 +144,9 @@ sql alter table tb1_tg2 set tag t1 = false sql alter table tb1_tg2 set tag t2 = 'binary2' sql alter table tb1_tg2 set tag t3 = '涛思' sql reset query cache + sql select * from stb1_tg +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 if $rows != 1 then return -1 endi @@ -164,7 +168,9 @@ endi if $data07 != -6 then return -1 endi + sql select * from stb2_tg +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 if $data02 != 0 then return -1 endi @@ -185,7 +191,9 @@ sql alter table tb1_tg2 set tag t1 = NULL sql alter table tb1_tg2 set tag t2 = NULL sql alter table tb1_tg2 set tag t3 = NULL sql reset query cache + sql select * from stb1_tg +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 if $rows != 1 then return -1 endi @@ -207,12 +215,12 @@ endi if $data07 != NULL then return -1 endi + sql select * from stb2_tg +print ===> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 if $data02 != NULL then return -1 endi - -print $data03 if $data03 != NULL then return -1 endi diff --git a/tests/script/tsim/parser/single_row_in_tb_query.sim b/tests/script/tsim/parser/single_row_in_tb_query.sim index acf85ea692..422756b798 100644 --- a/tests/script/tsim/parser/single_row_in_tb_query.sim +++ b/tests/script/tsim/parser/single_row_in_tb_query.sim @@ -1,4 +1,3 @@ -sleep 100 sql connect $dbPrefix = sr_db @@ -113,7 +112,7 @@ if $data02 != 2 then endi #### query a STable and using where clause -sql select first(ts,c1), last(ts,c1), spread(c1) from $stb where ts >= $ts0 and ts < '2018-09-20 00:00:00.000' group by t1 +sql select first(ts,c1), last(ts,c1), spread(c1), t1 from $stb where ts >= $ts0 and ts < '2018-09-20 00:00:00.000' group by t1 if $rows != 1 then return -1 endi @@ -137,7 +136,7 @@ if $data05 != 1 then return -1 endi -sql select first(c1), last(c1) from $stb where ts >= $ts0 and ts < '2018-09-20 00:00:00.000' interval(1d) group by t1 +sql select _wstart, first(c1), last(c1) from sr_stb where ts >= 1537146000000 and ts < '2018-09-20 00:00:00.000' partition by t1 interval(1d) if $rows != 1 then return -1 endi @@ -151,7 +150,7 @@ if $data02 != 1 then return -1 endi -sql select max(c1), min(c1), sum(c1), avg(c1), count(c1) from $stb where c1 > 0 group by t1 +sql select max(c1), min(c1), sum(c1), avg(c1), count(c1), t1 from $stb where c1 > 0 group by t1 if $rows != 1 then return -1 endi @@ -174,7 +173,7 @@ if $data05 != 1 then return -1 endi -sql select first(ts,c1), last(ts,c1) from $tb1 where ts >= $ts0 and ts < '2018-09-20 00:00:00.000' interval(1d) +sql select _wstart, first(ts,c1), last(ts,c1) from $tb1 where ts >= $ts0 and ts < '2018-09-20 00:00:00.000' interval(1d) if $rows != 1 then return -1 endi diff --git a/tests/script/tsim/parser/sliding.sim b/tests/script/tsim/parser/sliding.sim index 18d7bda8a1..6edab1f4a7 100644 --- a/tests/script/tsim/parser/sliding.sim +++ b/tests/script/tsim/parser/sliding.sim @@ -58,8 +58,6 @@ while $i < $tbNum $tstart = 946656000000 endw -sleep 100 - $i1 = 1 $i2 = 0 @@ -76,425 +74,358 @@ $ts1 = $tb1 . .ts $ts2 = $tb2 . .ts print ===============================interval_sliding query -sql select count(*) from sliding_tb0 interval(30s) sliding(30s); +sql select _wstart, count(*) from sliding_tb0 interval(30s) sliding(30s); if $row != 10 then return -1 endi - if $data00 != @00-01-01 00:00:00.000@ then return -1 endi - if $data01 != 1000 then return -1 endi - if $data10 != @00-01-01 00:00:30.000@ then return -1 endi - if $data11 != 1000 then return -1 endi -sql select stddev(c1) from sliding_tb0 interval(10a) sliding(10a) +sql select _wstart, stddev(c1) from sliding_tb0 interval(10a) sliding(10a); if $row != 10000 then return -1 endi - if $data00 != @00-01-01 00:00:00.000@ then return -1 endi - if $data01 != 0.000000000 then return -1 endi - if $data90 != @00-01-01 00:00:00.270@ then return -1 endi - if $data91 != 0.000000000 then return -1 endi -sql select stddev(c1),count(c2),first(c3),last(c4) from sliding_tb0 interval(10a) sliding(10a) order by ts desc; +sql select _wstart, stddev(c1),count(c2),first(c3),last(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc; if $row != 10000 then return -1 endi - if $data00 != @00-01-01 00:04:59.970@ then return -1 endi - if $data01 != 0.000000000 then return -1 endi - if $data02 != 1 then return -1 endi - if $data03 != 99 then return -1 endi - if $data04 != 99 then return -1 endi - if $data90 != @00-01-01 00:04:59.700@ then return -1 endi - if $data91 != 0.000000000 then return -1 endi - if $data92 != 1 then return -1 endi - if $data93 != 90 then return -1 endi - if $data94 != 90 then return -1 endi -sql select count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by ts asc; +sql select _wstart, count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by _wstart asc; if $row != 32 then return -1 endi - if $data00 != @99-12-31 23:59:40.000@ then print expect 12-31 23:59:40.000, actual: $data00 return -1 endi - if $data01 != 334 then return -1 endi - if $data02 != 33 then return -1 endi -sql select count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by ts asc; +sql select _wstart, count(c2),stddev(c3),first(c4),last(c4) from sliding_tb0 where ts>'2000-01-01 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc; if $row != 2 then return -1 endi - if $data04 != 99 then return -1 endi - if $data01 != 999 then return -1 endi - if $data02 != 28.837977152 then return -1 endi #interval offset + limit -sql select count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by ts desc limit 10 offset 990; +sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) sliding(10a) order by _wstart desc limit 10 offset 990; if $row != 10 then return -1 endi - if $data00 != @00-01-01 00:04:30.270@ then return -1 endi - if $data01 != 1 then return -1 endi - if $data02 != 9 then return -1 endi - if $data03 != 0.000000000 then return -1 endi - if $data90 != @00-01-01 00:04:30.000@ then return -1 endi - if $data91 != 1 then return -1 endi - if $data92 != 0 then return -1 endi - if $data93 != 0.000000000 then return -1 endi #interval offset test -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(30s) order by ts asc limit 1000 offset 1; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 1; if $row != 9 then return -1 endi - if $data00 != @00-01-01 00:00:30.000@ then return -1 endi - if $data01 != 1000 then return -1 endi - if $data02 != 99 then return -1 endi - if $data80 != @00-01-01 00:04:30.000@ then return -1 endi - if $data81 != 1000 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by ts asc limit 1000 offset 0; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 where ts>'2000-1-1 0:0:0' and ts<'2000-1-1 0:0:31' interval(30s) sliding(30s) order by _wstart asc limit 1000 offset 0; if $row != 2 then return -1 endi - if $data00 != @00-01-01 00:00:00.000@ then return -1 endi - if $data01 != 999 then return -1 endi - if $data02 != 99 then return -1 endi - if $data03 != 28.837977152 then return -1 endi - if $data10 != @00-01-01 00:00:30.000@ then return -1 endi - if $data11 != 34 then return -1 endi - if $data12 != 33 then return -1 endi - if $data13 != 9.810708435 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 1; +sql select _wstart, count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 1; if $row != 15 then return -1 endi - if $data00 != @00-01-01 00:00:00.000@ then return -1 endi - if $data01 != 1000 then return -1 endi - if $data02 != 99 then return -1 endi - if $data03 != 28.866070048 then return -1 endi - if $data90 != @00-01-01 00:03:00.000@ then return -1 endi - if $data91 != 1000 then return -1 endi - if $data92 != 99 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 5; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 5; if $row != 11 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 6; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 6; if $row != 10 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 7; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 7; if $row != 9 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 8; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 8; if $row != 8 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 9; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 9; if $row != 7 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 10; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 10; if $row != 6 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 11; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 11; if $row != 5 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 12; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 12; if $row != 4 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 13; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 13; if $row != 3 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 14; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 14; if $row != 2 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 15; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 15; if $row != 1 then return -1 endi -sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 16; +sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by _wstart asc limit 100 offset 16; if $row != 0 then return -1 endi -sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by ts desc; +sql select _wstart, count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by _wstart desc; if $row != 10 then return -1 endi - #00-01-01 00:04:30.000| 10| 0| 0.000000000| 0.000000000| if $data00 != @00-01-01 00:04:30.000@ then return -1 endi - if $data01 != 10 then return -1 endi - if $data02 != 0 then return -1 endi - if $data03 != 0.000000000 then return -1 endi -sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 15; +#TODO +return + +sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 15; if $row != 1 then return -1 endi -sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 16; +sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by _wstart desc limit 1 offset 16; if $row != 0 then return -1 endi -sql select count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by ts desc limit 10 offset 2; +sql select _wstart, count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by _wstart desc limit 10 offset 2; if $data00 != @00-01-01 00:04:59.910@ then return -1 endi -sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts asc limit 1000; +sql select _wstart, count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by _wstart asc limit 1000; if $row != 3 then return -1 endi - if $data00 != @99-12-31 23:59:40.000@ then return -1 endi - if $data02 != 9.521904571 then return -1 endi - if $data05 != 33 then return -1 endi - if $data10 != @99-12-31 23:59:50.000@ then return -1 endi - if $data12 != 9.521904571 then return -1 endi - if $data15 != 33 then return -1 endi - if $data25 != 33 then return -1 endi -sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts desc limit 1000; +sql select _wstart, count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by _wstart desc limit 1000; if $row != 3 then return -1 endi - if $data00 != @00-01-01 00:00:00.000@ then print expect 00-01-01 00:00:00.000, actual: $data00 return -1 endi - if $data01 != 33 then return -1 endi - if $data02 != 9.521904571 then return -1 endi - if $data03 != 33 then return -1 endi - if $data10 != @99-12-31 23:59:50.000@ then return -1 endi - if $data11 != 33 then return -1 endi if $data12 != 9.521904571 then return -1 endi - if $data20 != @99-12-31 23:59:40.000@ then return -1 endi diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim index 4d05d4ced7..95150616d1 100644 --- a/tests/script/tsim/parser/union.sim +++ b/tests/script/tsim/parser/union.sim @@ -102,11 +102,11 @@ $i = 1 $tb = $tbPrefix . $i ## column type not identical -sql_error select count(*) as a from union_mt0 union all select avg(c1) as a from union_mt0 -sql_error select count(*) as a from union_mt0 union all select spread(c1) as a from union_mt0; +sql select count(*) as a from union_mt0 union all select avg(c1) as a from union_mt0 +sql select count(*) as a from union_mt0 union all select spread(c1) as a from union_mt0; ## union not supported -sql_error (select count(*) from union_mt0) union (select count(*) from union_mt0); +sql (select count(*) from union_mt0) union (select count(*) from union_mt0); ## column type not identical sql_error select c1 from union_mt0 limit 10 union all select c2 from union_tb1 limit 20; @@ -123,145 +123,114 @@ sql (((select c1 from union_tb0))) if $rows != 10000 then return -1 endi - if $data00 != 0 then return -1 endi - if $data10 != 1 then return -1 endi -sql select 'ab' as options from union_tb1 limit 1 union all select 'dd' as options from union_tb0 limit 1; +sql (select 'ab' as options from union_tb1 limit 1) union all (select 'dd' as options from union_tb0 limit 1) order by options; if $rows != 2 then return -1 endi - if $data00 != @ab@ then return -1 endi - if $data10 != @dd@ then return -1 endi - -sql select 'ab' as options from union_tb1 limit 1 union all select '1234567' as options from union_tb0 limit 1; +sql (select 'ab12345' as options from union_tb1 limit 1) union all (select '1234567' as options from union_tb0 limit 1) order by options desc; if $rows != 2 then return -1 endi - -if $data00 != @ab@ then +if $data00 != @ab12345@ then return -1 endi - if $data10 != @1234567@ then return -1 endi - # mixed order -sql select ts, c1 from union_tb1 order by ts asc limit 10 union all select ts, c1 from union_tb0 order by ts desc limit 2 union all select ts, c1 from union_tb2 order by ts asc limit 10 +sql (select ts, c1 from union_tb1 order by ts asc limit 10) union all (select ts, c1 from union_tb0 order by ts desc limit 2) union all (select ts, c1 from union_tb2 order by ts asc limit 10) order by ts if $rows != 22 then return -1 endi - if $data00 != @20-01-05 13:51:24.000@ then return -1 endi - if $data01 != 0 then return -1 endi - -if $data10 != @20-01-05 13:52:24.000@ then +if $data10 != @20-01-05 13:51:24.000@ then return -1 endi - -if $data11 != 1 then +if $data11 != 0 then return -1 endi - -if $data90 != @20-01-05 14:00:24.000@ then +print $data90 $data91 +if $data90 != @20-01-05 13:55:24.000@ then return -1 endi - -if $data91 != 9 then +if $data91 != 4 then return -1 endi # different sort order # super table & normal table mixed up -sql select c3 from union_tb0 limit 2 union all select sum(c1) as c3 from union_mt0; +sql (select c3 from union_tb0 limit 2) union all (select sum(c1) as c3 from union_mt0) order by c3; if $rows != 3 then return -1 endi - if $data00 != 0 then return -1 endi - if $data10 != 1 then return -1 endi - if $data20 != 4950000 then return -1 endi # type compatible -sql select c3 from union_tb0 limit 2 union all select sum(c1) as c3 from union_tb1; +sql (select c3 from union_tb0 limit 2) union all (select sum(c1) as c3 from union_tb1) order by c3; if $rows != 3 then return -1 endi - if $data00 != 0 then return -1 endi - if $data10 != 1 then return -1 endi - if $data20 != 495000 then return -1 endi # two join subclause -sql select count(*) as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1.ts union all select union_tb0.c3 as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1.ts limit 10 +sql (select count(*) as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1.ts) union all (select union_tb0.c3 as c from union_tb0, union_tb1 where union_tb0.ts=union_tb1.ts limit 10) order by c desc if $rows != 11 then return -1 endi - if $data00 != 10000 then return -1 endi - -if $data10 != 0 then +if $data10 != 9 then return -1 endi - -if $data20 != 1 then +if $data20 != 8 then return -1 endi - -if $data90 != 8 then +if $data90 != 1 then return -1 endi print ===========================================tags union # two super table tag union, limit is not active during retrieve tags query -sql select t1 from union_mt0 union all select t1 from union_mt0 -if $rows != 20 then - return -1 -endi - -if $data00 != 0 then - return -1 -endi - -if $data90 != 9 then +sql (select t1 from union_mt0) union all (select t1 from union_mt0) +if $rows != 200000 then return -1 endi @@ -271,39 +240,35 @@ endi #endi #========================================== two super table join subclause print ================two super table join subclause -sql select avg(union_mt0.c1) as c from union_mt0 interval(1h) limit 10 union all select union_mt1.ts, union_mt1.c1/1.0 as c from union_mt0, union_mt1 where union_mt1.ts=union_mt0.ts and union_mt1.t1=union_mt0.t1 limit 5; +sql (select _wstart as ts, avg(union_mt0.c1) as c from union_mt0 interval(1h) limit 10) union all (select union_mt1.ts, union_mt1.c1/1.0 as c from union_mt0, union_mt1 where union_mt1.ts=union_mt0.ts and union_mt1.t1=union_mt0.t1 limit 5); print the rows value is: $rows - if $rows != 15 then return -1 endi # first subclause are empty -sql select count(*) as c from union_tb0 where ts > now + 3650d union all select sum(c1) as c from union_tb1; +sql (select count(*) as c from union_tb0 where ts > now + 3650d) union all (select sum(c1) as c from union_tb1); if $rows != 1 then return -1 endi - if $data00 != 495000 then return -1 endi # all subclause are empty -sql select c1 from union_tb0 limit 0 union all select c1 from union_tb1 where ts>'2021-1-1 0:0:0' +sql (select c1 from union_tb0 limit 0) union all (select c1 from union_tb1 where ts>'2021-1-1 0:0:0') if $rows != 0 then return -1 endi # middle subclause empty -sql select c1 from union_tb0 limit 1 union all select c1 from union_tb1 where ts>'2030-1-1 0:0:0' union all select last(c1) as c1 from union_tb1; +sql (select c1 from union_tb0 limit 1) union all (select c1 from union_tb1 where ts>'2030-1-1 0:0:0' union all select last(c1) as c1 from union_tb1) order by c1; if $rows != 2 then return -1 endi - if $data00 != 0 then return -1 endi - if $data10 != 99 then return -1 endi @@ -319,141 +284,90 @@ sql (select ts, c1 from union_mt0 limit 1) union all (select ts, c1 from union_m if $rows != 2 then return -1 endi - if $data00 != @20-01-05 13:51:24.000@ then return -1 endi - if $data01 != 0 then return -1 endi - if $data10 != @20-01-05 13:51:24.000@ then return -1 endi - if $data11 != 0 then return -1 endi # two aggregated functions for super tables -sql select sum(c1) as a from union_mt0 interval(1s) limit 9 union all select ts, max(c3) as a from union_mt0 limit 2; +sql (select _wstart as ts, sum(c1) as a from union_mt0 interval(1s) limit 9) union all (select ts, max(c3) as a from union_mt0 limit 2) order by ts; if $rows != 10 then return -1 endi - if $data00 != @20-01-05 13:51:24.000@ then return -1 endi - if $data01 != 0 then return -1 endi - if $data10 != @20-01-05 13:52:24.000@ then return -1 endi - if $data11 != 10 then return -1 endi - if $data20 != @20-01-05 13:53:24.000@ then return -1 endi - if $data21 != 20 then return -1 endi - if $data90 != @20-01-05 15:30:24.000@ then return -1 endi - if $data91 != 99 then return -1 endi #================================================================================================= # two aggregated functions for normal tables -sql select sum(c1) as a from union_tb0 limit 1 union all select sum(c3) as a from union_tb1 limit 2; +sql (select sum(c1) as a from union_tb0 limit 1) union all (select sum(c3) as a from union_tb1 limit 2); if $rows != 2 then return -1 endi - if $data00 != 495000 then return -1 endi - if $data10 != 495000 then return -1 endi # two super table query + interval + limit -sql select ts, first(c3) as a from union_mt0 limit 1 union all select sum(c3) as a from union_mt0 interval(1h) limit 1; +sql (select ts, first(c3) as a from union_mt0 limit 1) union all (select _wstart as ts, sum(c3) as a from union_mt0 interval(1h) limit 1) order by ts desc; if $rows != 2 then return -1 endi - if $data00 != @20-01-05 13:51:24.000@ then return -1 endi - if $data01 != 0 then return -1 endi - if $data10 != @20-01-05 13:00:00.000@ then return -1 endi - if $data11 != 360 then return -1 endi -sql select server_status() union all select server_status() -if $rows != 2 then - return -1 -endi - -if $data00 != 1 then - return -1 -endi - -if $data10 != 1 then - return -1 -endi - -sql select client_version() union all select server_version() -if $rows != 2 then - return -1 -endi - -sql select database() union all select database() -if $rows != 2 then - return -1 -endi - -if $data00 != @union_db0@ then - return -1 -endi - -if $data10 != @union_db0@ then - return -1 -endi - -sql select 'aaa' as option from union_tb1 where c1 < 0 limit 1 union all select 'bbb' as option from union_tb0 limit 1 +sql (select 'aaa' as option from union_tb1 where c1 < 0 limit 1) union all (select 'bbb' as option from union_tb0 limit 1) if $rows != 1 then return -1 endi - if $data00 != @bbb@ then return -1 endi - -sql_error show tables union all show tables -sql_error show stables union all show stables -sql_error show databases union all show databases +sql_error (show tables) union all (show tables) +sql_error (show stables) union all (show stables) +sql_error (show databases) union all (show databases) system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/union_sysinfo.sim b/tests/script/tsim/parser/union_sysinfo.sim new file mode 100644 index 0000000000..ea45dc68e1 --- /dev/null +++ b/tests/script/tsim/parser/union_sysinfo.sim @@ -0,0 +1,35 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql (select server_status()) union all (select server_status()) +if $rows != 2 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +sql (select client_version()) union all (select server_version()) +if $rows != 2 then + return -1 +endi + +sql (select database()) union all (select database()) +if $rows != 2 then + return -1 +endi +if $data00 != @union_db0@ then + return -1 +endi +if $data10 != @union_db0@ then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index a6ee5951a0..d4e575801c 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -391,13 +391,13 @@ if $data02 != 4 then return -1 endi -if $data03 != 14 then - print ======$data03 +if $data03 != 50 then + print ======$data03 != 50 return -1 endi -if $data04 != 4 then - print ======$data04 +if $data04 != 20 then + print ======$data04 != 20 return -1 endi @@ -421,13 +421,13 @@ if $data12 != 4 then return -1 endi -if $data13 != 10 then - print ======$data13 +if $data13 != 46 then + print ======$data13 != 46 return -1 endi -if $data14 != 3 then - print ======$data14 +if $data14 != 20 then + print ======$data14 != 20 return -1 endi diff --git a/tests/script/tsim/tag/commit.sim b/tests/script/tsim/tag/commit.sim index 18128fc464..95ab2dbc7f 100644 --- a/tests/script/tsim/tag/commit.sim +++ b/tests/script/tsim/tag/commit.sim @@ -241,7 +241,7 @@ if $data04 != 3 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol4 sql alter table $mt drop tag tgcol2 sql alter table $mt drop tag tgcol3 sql alter table $mt add tag tgcol5 binary(10) diff --git a/tests/script/tsim/valgrind/checkError5.sim b/tests/script/tsim/valgrind/checkError5.sim index f3d418cfd1..6eef185fd3 100644 --- a/tests/script/tsim/valgrind/checkError5.sim +++ b/tests/script/tsim/valgrind/checkError5.sim @@ -95,6 +95,16 @@ sql select * from tb sql insert into db.ctb values(now+3s, 2, 3, 4) sql select * from db.stb +sql alter table db.stb add tag t4 bigint +sql select * from db.stb +sql select * from db.stb +sql_error create table db.ctb2 using db.stb tags(101, "102") +sql create table db.ctb2 using db.stb tags(101, 102, "103", 104) +sql insert into db.ctb2 values(now, 1, 2, 3) + +print =============== step6: query data +sql select * from db.stb where tbname = 'ctb2'; + _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT print =============== check From eb2dcbdc46ddde73a6a359ecddad2b864f9610a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:19:59 +0800 Subject: [PATCH 13/36] other: merge 3.0 --- docs/zh/02-intro.md | 2 +- docs/zh/05-get-started/03-package.md | 154 +++++++++++++++++-------- docs/zh/05-get-started/06-first-use.md | 135 ---------------------- docs/zh/10-deployment/01-deploy.md | 12 +- docs/zh/13-operation/01-pkg-install.md | 78 ++++++++++++- 5 files changed, 185 insertions(+), 196 deletions(-) delete mode 100644 docs/zh/05-get-started/06-first-use.md diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md index 673c2e96b6..191e1cbcc2 100644 --- a/docs/zh/02-intro.md +++ b/docs/zh/02-intro.md @@ -52,7 +52,7 @@ TDengine的主要功能如下： 采用 TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。表现在几个方面： 1. 由于其超强性能，它能将系统需要的计算资源和存储资源大幅降低 -2. 因为采用 SQL 接口，能与众多第三放软件无缝集成，学习迁移成本大幅下降 +2. 因为采用 SQL 接口，能与众多第三方软件无缝集成，学习迁移成本大幅下降 3. 因为其 All In One 的特性，系统复杂度降低，能降研发成本 4. 因为运维维护简单，运营维护成本能大幅降低 diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index a21066e0cd..6ac7567a05 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -1,6 +1,6 @@ --- sidebar_label: 安装包 -title: 使用安装包安装和卸载 +title: 使用安装包立即开始 --- import Tabs from "@theme/Tabs"; @@ -169,72 +169,128 @@ install.sh 安装脚本在执行过程中，会通过命令行交互界面询问 ::: -## 卸载 +## 启动 - - - -内容 TBD - - - - -卸载命令如下: - -``` -$ sudo dpkg -r tdengine -(Reading database ... 137504 files and directories currently installed.) -Removing tdengine (2.4.0.7) ... -TDengine is removed successfully! +安装后，请使用 `systemctl` 命令来启动 TDengine 的服务进程。 +```bash +systemctl start taosd ``` - +检查服务是否正常工作： - - -卸载命令如下: - -``` -$ sudo rpm -e tdengine -TDengine is removed successfully! +```bash +systemctl status taosd ``` - - - - -卸载命令如下: +如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： ``` -$ rmtaos -Nginx for TDengine is running, stopping it... -TDengine is removed successfully! - -taosKeeper is removed successfully! +Active: active (running) ``` - - +如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： + +``` +Active: inactive (dead) +``` + +如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。 + +systemctl 命令汇总： + +- 启动服务进程：`systemctl start taosd` + +- 停止服务进程：`systemctl stop taosd` + +- 重启服务进程：`systemctl restart taosd` + +- 查看服务状态：`systemctl status taosd` :::info -- TDengine 提供了多种安装包，但最好不要在一个系统上同时使用 tar.gz 安装包和 deb 或 rpm 安装包。否则会相互影响，导致在使用时出现问题。 +- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。 +- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。 +- 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。 -- 对于 deb 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： +::: - ``` - $ sudo rm -f /var/lib/dpkg/info/tdengine* - ``` +## TDengine 命令行 (CLI) -然后再重新进行安装就可以了。 +为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可。 -- 对于 rpm 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： +```bash +taos +``` - ``` - $ sudo rpm -e --noscripts tdengine - ``` +如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下： -然后再重新进行安装就可以了。 +```cmd +taos> +``` -::: \ No newline at end of file +在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例： + +```sql +create database demo; +use demo; +create table t (ts timestamp, speed int); +insert into t values ('2019-07-15 00:00:00', 10); +insert into t values ('2019-07-15 01:00:00', 20); +select * from t; + ts | speed | +======================================== + 2019-07-15 00:00:00.000 | 10 | + 2019-07-15 01:00:00.000 | 20 | +Query OK, 2 row(s) in set (0.003128s) +``` + +除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../reference/taos-shell/) + +## 使用 taosBenchmark 体验写入速度 + +启动 TDengine 的服务，在 Linux 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）： + +```bash +taosBenchmark +``` + +该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。 + +这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。 + +taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。 + +## 使用 TDengine CLI 体验查询速度 + +使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。 + +查询超级表下记录总条数： + +```sql +taos> select count(*) from test.meters; +``` + +查询 1 亿条记录的平均值、最大值、最小值等： + +```sql +taos> select avg(current), max(voltage), min(phase) from test.meters; +``` + +查询 location="California.SanFrancisco" 的记录总条数： + +```sql +taos> select count(*) from test.meters where location="California.SanFrancisco"; +``` + +查询 groupId=10 的所有记录的平均值、最大值、最小值等： + +```sql +taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10; +``` + +对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： + +```sql +taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); +``` \ No newline at end of file diff --git a/docs/zh/05-get-started/06-first-use.md b/docs/zh/05-get-started/06-first-use.md deleted file mode 100644 index 927ce0a1bd..0000000000 --- a/docs/zh/05-get-started/06-first-use.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -sidebar_label: 开始使用 -title: 快速体验 TDengine ---- - -import Tabs from "@theme/Tabs"; -import TabItem from "@theme/TabItem"; -import PkgInstall from "./\_pkg_install.mdx"; -import AptGetInstall from "./\_apt_get_install.mdx"; - -## 启动 - -安装后，请使用 `systemctl` 命令来启动 TDengine 的服务进程。 - -```bash -systemctl start taosd -``` - -检查服务是否正常工作： - -```bash -systemctl status taosd -``` - -如果服务进程处于活动状态，则 status 指令会显示如下的相关信息： - -``` -Active: active (running) -``` - -如果后台服务进程处于停止状态，则 status 指令会显示如下的相关信息： - -``` -Active: inactive (dead) -``` - -如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。 - -systemctl 命令汇总： - -- 启动服务进程：`systemctl start taosd` - -- 停止服务进程：`systemctl stop taosd` - -- 重启服务进程：`systemctl restart taosd` - -- 查看服务状态：`systemctl status taosd` - -:::info - -- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。 -- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。 -- 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。 - -::: - -## TDengine 命令行 (CLI) - -为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可。 - -```bash -taos -``` - -如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下： - -```cmd -taos> -``` - -在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例： - -```sql -create database demo; -use demo; -create table t (ts timestamp, speed int); -insert into t values ('2019-07-15 00:00:00', 10); -insert into t values ('2019-07-15 01:00:00', 20); -select * from t; - ts | speed | -======================================== - 2019-07-15 00:00:00.000 | 10 | - 2019-07-15 01:00:00.000 | 20 | -Query OK, 2 row(s) in set (0.003128s) -``` - -除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../reference/taos-shell/) - -## 使用 taosBenchmark 体验写入速度 - -启动 TDengine 的服务，在 Linux 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）： - -```bash -taosBenchmark -``` - -该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。 - -这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。 - -taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。 - -## 使用 TDengine CLI 体验查询速度 - -使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。 - -查询超级表下记录总条数： - -```sql -taos> select count(*) from test.meters; -``` - -查询 1 亿条记录的平均值、最大值、最小值等： - -```sql -taos> select avg(current), max(voltage), min(phase) from test.meters; -``` - -查询 location="California.SanFrancisco" 的记录总条数： - -```sql -taos> select count(*) from test.meters where location="California.SanFrancisco"; -``` - -查询 groupId=10 的所有记录的平均值、最大值、最小值等： - -```sql -taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10; -``` - -对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： - -```sql -taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); -``` diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index ed2d5653f5..bfd4804e30 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -55,6 +55,8 @@ fqdn h1.taosdata.com // 配置本数据节点的端口号，缺省是 6030 serverPort 6030 +``` + 一定要修改的参数是 firstEp 和 fqdn。在每个数据节点，firstEp 需全部配置成一样，但 fqdn 一定要配置成其所在数据节点的值。其他参数可不做任何修改，除非你很清楚为什么要修改。 加入到集群中的数据节点 dnode，下表中涉及集群相关的参数必须完全相同，否则不能成功加入到集群中。 @@ -68,12 +70,9 @@ serverPort 6030 ## 启动集群 -### 启动第一个数据节点 - 按照《立即开始》里的步骤，启动第一个数据节点，例如 h1.taosdata.com，然后执行 taos，启动 taos shell，从 shell 里执行命令“SHOW DNODES”，如下所示： ``` - Welcome to the TDengine shell from Linux, Client Version:3.0.0.0 Copyright (c) 2022 by TAOS Data, Inc. All rights reserved. @@ -85,15 +84,12 @@ id | endpoint | vnodes | support_vnodes | status | create_time | note | 1 | h1.taosdata.com:6030 | 0 | 1024 | ready | 2022-07-16 10:50:42.673 | | Query OK, 1 rows affected (0.007984s) -taos> -taos> - -```` +``` 上述命令里，可以看到刚启动的数据节点的 End Point 是：h1.taos.com:6030，就是这个新集群的 firstEp。 -### 添加数据节点 +## 添加数据节点 将后续的数据节点添加到现有集群，具体有以下几步： diff --git a/docs/zh/13-operation/01-pkg-install.md b/docs/zh/13-operation/01-pkg-install.md index 41daffc1b7..f814ee70b7 100644 --- a/docs/zh/13-operation/01-pkg-install.md +++ b/docs/zh/13-operation/01-pkg-install.md @@ -8,9 +8,11 @@ import TabItem from "@theme/TabItem"; 本节将介绍一些关于安装和卸载更深层次的内容，以及升级的注意事项。 -## 安装和卸载 +## 安装 + +关于安装，请参考 [使用安装包立即开始](../get-started/package) + -关于安装和卸载，请参考 [安装和卸载](../get-started/package) ## 安装目录说明 @@ -40,6 +42,76 @@ lrwxrwxrwx 1 root root 13 Feb 22 09:34 log -> /var/log/taos/ - /usr/local/taos/driver 目录下的动态库文件，会软链接到 /usr/lib 目录下； - /usr/local/taos/include 目录下的头文件，会软链接到到 /usr/include 目录下； +## 卸载 + + + + +内容 TBD + + + + +卸载命令如下: + +``` +$ sudo dpkg -r tdengine +(Reading database ... 137504 files and directories currently installed.) +Removing tdengine (2.4.0.7) ... +TDengine is removed successfully! + +``` + + + + + +卸载命令如下: + +``` +$ sudo rpm -e tdengine +TDengine is removed successfully! +``` + + + + + +卸载命令如下: + +``` +$ rmtaos +Nginx for TDengine is running, stopping it... +TDengine is removed successfully! + +taosKeeper is removed successfully! +``` + + + + +:::info + +- TDengine 提供了多种安装包，但最好不要在一个系统上同时使用 tar.gz 安装包和 deb 或 rpm 安装包。否则会相互影响，导致在使用时出现问题。 + +- 对于 deb 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： + + ``` + $ sudo rm -f /var/lib/dpkg/info/tdengine* + ``` + +然后再重新进行安装就可以了。 + +- 对于 rpm 包安装后，如果安装目录被手工误删了部分，出现卸载、或重新安装不能成功。此时，需要清除 TDengine 包的安装信息，执行如下命令： + + ``` + $ sudo rpm -e --noscripts tdengine + ``` + +然后再重新进行安装就可以了。 + +::: + ## 卸载和更新文件说明 卸载安装包的时候，将保留配置文件、数据库文件和日志文件，即 /etc/taos/taos.cfg 、 /var/lib/taos 、 /var/log/taos 。如果用户确认后不需保留，可以手工删除，但一定要慎重，因为删除后，数据将永久丢失，不可以恢复！ @@ -64,4 +136,4 @@ lrwxrwxrwx 1 root root 13 Feb 22 09:34 log -> /var/log/taos/ :::warning TDengine 不保证低版本能够兼容高版本的数据，所以任何时候都不推荐降级 -::: \ No newline at end of file +::: From 0a37d4641fbfb10c6cec8197caa32d5fde0b9832 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:20:33 +0800 Subject: [PATCH 14/36] other: merge 3.0 --- tests/script/test.sh | 1 + tests/system-test/1-insert/delete_data.py | 21 +- tests/system-test/1-insert/update_data.py | 38 +++- .../1-insert/update_data_muti_rows.py | 179 ++++++++++++++++++ tests/system-test/2-query/Now.py | 16 +- tests/system-test/2-query/count.py | 10 +- .../2-query/distribute_agg_apercentile.py | 24 +-- .../system-test/2-query/distribute_agg_avg.py | 44 ++--- .../2-query/distribute_agg_count.py | 52 ++--- .../system-test/2-query/distribute_agg_max.py | 56 +++--- .../system-test/2-query/distribute_agg_min.py | 56 +++--- .../2-query/distribute_agg_spread.py | 48 ++--- .../system-test/2-query/distribute_agg_sum.py | 44 ++--- tests/system-test/2-query/irate.py | 10 +- tests/system-test/2-query/last.py | 6 +- tests/system-test/2-query/log.py | 118 ++++++------ .../2-query/query_cols_tags_and_or.py | 30 +-- 17 files changed, 489 insertions(+), 264 deletions(-) create mode 100644 tests/system-test/1-insert/update_data_muti_rows.py diff --git a/tests/script/test.sh b/tests/script/test.sh index 1cfe8dd6f5..0ffe8cf8f1 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -84,6 +84,7 @@ echo "SIM_DIR : $SIM_DIR" echo "CODE_DIR : $CODE_DIR" echo "CFG_DIR : $CFG_DIR" +rm -rf $SIM_DIR/* rm -rf $LOG_DIR rm -rf $CFG_DIR diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 4c1426d0b1..1eb270d997 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -214,6 +214,24 @@ class TDTestCase: tdSql.checkRows((row_num-i)*tb_num) for j in range(tb_num): self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + for i in range(row_num): + tdSql.execute(f'delete from {tbname} where ts between {self.ts} and {self.ts+i}') + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num - i-1) + self.insert_base_data(col_type,tbname,row_num,base_data) + elif tb_type == 'stb': + tdSql.checkRows(tb_num*(row_num - i-1)) + for j in range(tb_num): + self.insert_base_data(col_type,f'{tbname}_{j}',row_num,base_data) + tdSql.execute(f'delete from {tbname} where ts between {self.ts+i+1} and {self.ts}') + tdSql.query(f'select {col_name} from {tbname}') + if tb_type == 'ntb' or tb_type == 'ctb': + tdSql.checkRows(row_num) + elif tb_type == 'stb': + tdSql.checkRows(tb_num*row_num) def delete_error(self,tbname,column_name,column_type,base_data): for error_list in ['',f'ts = {self.ts} and',f'ts = {self.ts} or']: if 'binary' in column_type.lower(): @@ -221,7 +239,8 @@ class TDTestCase: elif 'nchar' in column_type.lower(): tdSql.error(f'''delete from {tbname} where {error_list} {column_name} ="{base_data['nchar']}"''') else: - tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + tdSql.error(f'delete from {tbname} where {error_list} {column_name} = {base_data[column_type]}') + def delete_data_ntb(self): tdSql.execute(f'create database if not exists {self.dbname}') tdSql.execute(f'use {self.dbname}') diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index 27e1559d7e..29d2a91d28 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -13,6 +13,7 @@ import random import string +from datetime import datetime from util import constant from util.log import * from util.cases import * @@ -55,7 +56,7 @@ class TDTestCase: else: tdLog.exit(f'{col_name} data check failure') elif col_type.lower() == 'timestamp': - tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) + tdSql.checkEqual(str(tdSql.queryResult[0][0]),str(datetime.fromtimestamp(value/1000).strftime("%Y-%m-%d %H:%M:%S.%f"))) else: tdSql.checkEqual(tdSql.queryResult[0][0],value) def update_and_check_data(self,tbname,col_name,col_type,value,dbname): @@ -81,39 +82,63 @@ class TDTestCase: if col_type.lower() == 'double': for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.DOUBLE_MIN,1.1*constant.DOUBLE_MAX]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'float': for error_value in [tdCom.getLongName(self.str_length),True,False,1.1*constant.FLOAT_MIN,1.1*constant.FLOAT_MAX]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif 'binary' in col_type.lower() or 'nchar' in col_type.lower(): for error_value in [tdCom.getLongName(str_length)]: tdSql.error(f'insert into {tbname} values({self.ts},"{error_value}")') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'bool': for error_value in [tdCom.getLongName(self.str_length)]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'tinyint': for error_value in [constant.TINYINT_MIN-1,constant.TINYINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'smallint': for error_value in [constant.SMALLINT_MIN-1,constant.SMALLINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'int': for error_value in [constant.INT_MIN-1,constant.INT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'bigint': for error_value in [constant.BIGINT_MIN-1,constant.BIGINT_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'tinyint unsigned': for error_value in [constant.TINYINT_UN_MIN-1,constant.TINYINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'smallint unsigned': for error_value in [constant.SMALLINT_UN_MIN-1,constant.SMALLINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'int unsigned': for error_value in [constant.INT_UN_MIN-1,constant.INT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') elif col_type.lower() == 'bigint unsigned': for error_value in [constant.BIGINT_UN_MIN-1,constant.BIGINT_UN_MAX+1,random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX),tdCom.getLongName(self.str_length),True,False]: - tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + tdSql.error(f'insert into {tbname} values({self.ts},{error_value})') + if tb_type == 'ctb': + tdSql.error(f'insert into {stbname} values({self.ts},{error_value})') tdSql.execute(f'drop table {tbname}') if tb_type == 'ctb': tdSql.execute(f'drop table {stbname}') @@ -218,8 +243,11 @@ class TDTestCase: self.error_check(self.ctbname,self.column_dict,'ctb',self.stbname) def run(self): - self.update_check() - self.update_check_error() + #!bug TD-17708 and TD-17709 + # for i in range(10): + self.update_check() + self.update_check_error() + # i+=1 def stop(self): tdSql.close() diff --git a/tests/system-test/1-insert/update_data_muti_rows.py b/tests/system-test/1-insert/update_data_muti_rows.py new file mode 100644 index 0000000000..e7da35426a --- /dev/null +++ b/tests/system-test/1-insert/update_data_muti_rows.py @@ -0,0 +1,179 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db_test' + self.ntbname = 'ntb' + self.stbname = 'stb' + self.rowNum = 10 + self.tbnum = 5 + self.ts = 1537146000000 + self.str_length = 20 + self.column_dict = { + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': f'binary({self.str_length})', + 'col13': f'nchar({self.str_length})' + } + self.tinyint_val = random.randint(constant.TINYINT_MIN,constant.TINYINT_MAX) + self.smallint_val = random.randint(constant.SMALLINT_MIN,constant.SMALLINT_MAX) + self.int_val = random.randint(constant.INT_MIN,constant.INT_MAX) + self.bigint_val = random.randint(constant.BIGINT_MIN,constant.BIGINT_MAX) + self.untingint_val = random.randint(constant.TINYINT_UN_MIN,constant.TINYINT_UN_MAX) + self.unsmallint_val = random.randint(constant.SMALLINT_UN_MIN,constant.SMALLINT_UN_MAX) + self.unint_val = random.randint(constant.INT_UN_MIN,constant.INT_MAX) + self.unbigint_val = random.randint(constant.BIGINT_UN_MIN,constant.BIGINT_UN_MAX) + self.float_val = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) + self.double_val = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) + self.bool_val = random.randint(0,2)%2 + self.binary_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.nchar_val = tdCom.getLongName(random.randint(0,self.str_length)) + self.data = { + 'tinyint':self.tinyint_val, + 'smallint':self.smallint_val, + 'int':self.int_val, + 'bigint':self.bigint_val, + 'tinyint unsigned':self.untingint_val, + 'smallint unsigned':self.unsmallint_val, + 'int unsigned':self.unint_val, + 'bigint unsigned':self.unbigint_val, + 'bool':self.bool_val, + 'float':self.float_val, + 'double':self.double_val, + 'binary':self.binary_val, + 'nchar':self.nchar_val + } + def update_data(self,dbname,tbname,tb_num,rows,values,col_type): + sql = f'insert into ' + for j in range(tb_num): + sql += f'{dbname}.{tbname}_{j} values' + for i in range(rows): + if 'binary' in col_type.lower() or 'nchar' in col_type.lower(): + sql += f'({self.ts+i},"{values}")' + else: + sql += f'({self.ts+i},{values})' + sql += ' ' + tdSql.execute(sql) + + def insert_data(self,col_type,tbname,rows,data): + for i in range(rows): + if col_type.lower() == 'tinyint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["tinyint"]})') + elif col_type.lower() == 'smallint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["smallint"]})') + elif col_type.lower() == 'int': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["int"]})') + elif col_type.lower() == 'bigint': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["bigint"]})') + elif col_type.lower() == 'tinyint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["tinyint unsigned"]})') + elif col_type.lower() == 'smallint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["smallint unsigned"]})') + elif col_type.lower() == 'int unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["int unsigned"]})') + elif col_type.lower() == 'bigint unsigned': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["bigint unsigned"]})') + elif col_type.lower() == 'bool': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["bool"]})') + elif col_type.lower() == 'float': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["float"]})') + elif col_type.lower() == 'double': + tdSql.execute(f'insert into {tbname} values({self.ts+i},{data["double"]})') + elif 'binary' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{data['binary']}")''') + elif 'nchar' in col_type.lower(): + tdSql.execute(f'''insert into {tbname} values({self.ts+i},"{data['nchar']}")''') + + def data_check(self,dbname,tbname,tbnum,rownum,data,col_name,col_type): + if 'binary' in col_type.lower(): + self.update_data(dbname,f'{tbname}',tbnum,rownum,data['binary'],col_type) + elif 'nchar' in col_type.lower(): + self.update_data(dbname,f'{tbname}',tbnum,rownum,data['nchar'],col_type) + else: + self.update_data(dbname,f'{tbname}',tbnum,rownum,data[col_type],col_type) + tdSql.execute(f'flush database {dbname}') + tdSql.execute('reset query cache') + for i in range(self.tbnum): + tdSql.query(f'select {col_name} from {dbname}.{tbname}_{i}') + for j in range(rownum): + if col_type.lower() == 'float' or col_type.lower() == 'double': + if abs(tdSql.queryResult[j][0] - data[col_type]) / data[col_type] <= 0.0001: + tdSql.checkEqual(tdSql.queryResult[j][0],tdSql.queryResult[j][0]) + elif 'binary' in col_type.lower(): + tdSql.checkEqual(tdSql.queryResult[j][0],data['binary']) + elif 'nchar' in col_type.lower(): + tdSql.checkEqual(tdSql.queryResult[j][0],data['nchar']) + else: + tdSql.checkEqual(tdSql.queryResult[j][0],data[col_type]) + def update_data_ntb(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + for i in range(self.tbnum): + tdSql.execute(f'create table {self.dbname}.{self.ntbname}_{i} (ts timestamp,{col_name} {col_type})') + for j in range(self.rowNum): + tdSql.execute(f'insert into {self.dbname}.{self.ntbname}_{i} values({self.ts+j},null)' ) + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute('reset query cache') + self.data_check(self.dbname,self.ntbname,self.tbnum,self.rowNum,self.data,col_name,col_type) + for i in range(self.tbnum): + tdSql.execute(f'drop table {self.ntbname}_{i}') + def update_data_ctb(self): + tdSql.execute(f'drop database if exists {self.dbname}') + tdSql.execute(f'create database {self.dbname}') + tdSql.execute(f'use {self.dbname}') + for col_name,col_type in self.column_dict.items(): + tdSql.execute(f'create table {self.dbname}.{self.stbname} (ts timestamp,{col_name} {col_type}) tags(t0 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.dbname}.{self.stbname}_{i} using {self.dbname}.{self.stbname} tags(1)') + for j in range(self.rowNum): + tdSql.execute(f'insert into {self.dbname}.{self.stbname}_{i} values({self.ts+j},null)' ) + tdSql.execute(f'flush database {self.dbname}') + tdSql.execute('reset query cache') + self.data_check(self.dbname,self.stbname,self.tbnum,self.rowNum,self.data,col_name,col_type) + tdSql.execute(f'drop table {self.stbname}') + def run(self): + self.update_data_ntb() + self.update_data_ctb() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py index 3caf632209..386c8b9d31 100644 --- a/tests/system-test/2-query/Now.py +++ b/tests/system-test/2-query/Now.py @@ -12,24 +12,24 @@ class TDTestCase: tdSql.init(conn.cursor(),True) self.setsql = TDSetSql() # name of normal table - self.ntbname = 'ntb' + self.ntbname = 'ntb' # name of stable - self.stbname = 'stb' + self.stbname = 'stb' # structure of column - self.column_dict = { + self.column_dict = { 'ts':'timestamp', 'c1':'int', 'c2':'float', 'c3':'double' } # structure of tag - self.tag_dict = { + self.tag_dict = { 't0':'int' } # number of child tables - self.tbnum = 2 + self.tbnum = 2 # values of tag,the number of values should equal to tbnum - self.tag_values = [ + self.tag_values = [ f'10', f'100' ] @@ -43,7 +43,7 @@ class TDTestCase: self.db_percision = ['ms','us','ns'] def tbtype_check(self,tb_type): if tb_type == 'normal table' or tb_type == 'child table': - tdSql.checkRows(len(self.values_list)) + tdSql.checkRows(len(self.values_list)) elif tb_type == 'stable': tdSql.checkRows(len(self.values_list) * self.tbnum) def data_check(self,tbname,tb_type): @@ -98,7 +98,7 @@ class TDTestCase: self.now_check_ntb() self.now_check_stb() - + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/count.py b/tests/system-test/2-query/count.py index c83ff43c51..e047225c1f 100644 --- a/tests/system-test/2-query/count.py +++ b/tests/system-test/2-query/count.py @@ -94,17 +94,15 @@ class TDTestCase: tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) for i in range(self.tbnum): tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[i]})') - #!TODO - # tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') - # tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) tdSql.query(f'select count(tbname) from {self.stbname}') tdSql.checkRows(0) tdSql.execute('flush database db') tdSql.query(f'select count(tbname) from {self.stbname}') tdSql.checkRows(0) - #!TODO - # tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') - # tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) + tdSql.query(f'SELECT count(*) from (select distinct tbname from {self.stbname})') + tdSql.checkEqual(tdSql.queryResult[0][0],self.tbnum) for i in range(self.tbnum): self.insert_data(self.column_dict,f'{self.stbname}_{i}',self.rowNum) self.count_query_stb(self.column_dict,self.tag_dict,self.stbname,self.tbnum,self.rowNum) diff --git a/tests/system-test/2-query/distribute_agg_apercentile.py b/tests/system-test/2-query/distribute_agg_apercentile.py index 632bda6bc6..eb5e8333c2 100644 --- a/tests/system-test/2-query/distribute_agg_apercentile.py +++ b/tests/system-test/2-query/distribute_agg_apercentile.py @@ -2,11 +2,11 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } @@ -18,7 +18,7 @@ class TDTestCase: self.ts = 1537146000000 def prepare_datas_of_distribute(self): - + # prepate datas for 20 tables distributed at different vgroups tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") @@ -89,17 +89,17 @@ class TDTestCase: vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -108,7 +108,7 @@ class TDTestCase: count+=1 if count < 2: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") - + def distribute_agg_query(self): # basic filter tdSql.query("select apercentile(c1 , 20) from stb1 where c1 is null") @@ -129,12 +129,12 @@ class TDTestCase: tdSql.query("select apercentile(c1,20) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all + # union all tdSql.query("select apercentile(c1,20) from stb1 union all select apercentile(c1,20) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,7.389181281) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") @@ -142,7 +142,7 @@ class TDTestCase: tdSql.execute(" create table tb1 using st tags(1) ") tdSql.execute(" create table tb2 using st tags(2) ") - + for i in range(10): ts = i*10 + self.ts tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") @@ -153,7 +153,7 @@ class TDTestCase: tdSql.checkData(0,0,9.000000000) tdSql.checkData(0,0,9.000000000) - # group by + # group by tdSql.execute(" use testdb ") tdSql.query(" select max(c1),c1 from stb1 group by t1 ") tdSql.checkRows(20) @@ -189,7 +189,7 @@ class TDTestCase: self.check_distribute_datas() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_avg.py b/tests/system-test/2-query/distribute_agg_avg.py index d23a597e92..2f449595bd 100644 --- a/tests/system-test/2-query/distribute_agg_avg.py +++ b/tests/system-test/2-query/distribute_agg_avg.py @@ -7,7 +7,7 @@ import platform class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } @@ -24,7 +24,7 @@ class TDTestCase: avg_sql = f"select avg({col_name}) from {tbname};" same_sql = f"select {col_name} from {tbname} where {col_name} is not null " - + tdSql.query(same_sql) pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): @@ -35,7 +35,7 @@ class TDTestCase: tdSql.checkData(0,0,pre_avg) def prepare_datas_of_distribute(self): - + # prepate datas for 20 tables distributed at different vgroups tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") @@ -106,17 +106,17 @@ class TDTestCase: vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -127,14 +127,14 @@ class TDTestCase: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") def check_avg_distribute_diff_vnode(self,col_name): - + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -143,7 +143,7 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - + avg_sql = f"select avg({col_name}) from stb1 where tbname in ({tbname_filters});" same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null " @@ -158,8 +158,8 @@ class TDTestCase: tdSql.checkData(0,0,pre_avg) def check_avg_status(self): - # check max function work status - + # check max function work status + tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] @@ -168,26 +168,26 @@ class TDTestCase: tdSql.query("desc stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_avg_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): self.check_avg_distribute_diff_vnode(colname) else: - # self.check_avg_distribute_diff_vnode(colname) # bug for tag + # self.check_avg_distribute_diff_vnode(colname) # bug for tag pass - + def distribute_agg_query(self): # basic filter tdSql.query(" select avg(c1) from stb1 ") @@ -211,7 +211,7 @@ class TDTestCase: tdSql.query("select avg(c1) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all + # union all tdSql.query("select avg(c1) from stb1 union all select avg(c1) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,14.086956522) @@ -220,7 +220,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,14.086956522) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") @@ -228,7 +228,7 @@ class TDTestCase: tdSql.execute(" create table tb1 using st tags(1) ") tdSql.execute(" create table tb2 using st tags(2) ") - + for i in range(10): ts = i*10 + self.ts tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") @@ -239,7 +239,7 @@ class TDTestCase: tdSql.checkData(0,0,4.500000000) tdSql.checkData(0,1,4.500000000) - # group by + # group by tdSql.execute(" use testdb ") # partition by tbname or partition by tag @@ -270,7 +270,7 @@ class TDTestCase: self.check_avg_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_count.py b/tests/system-test/2-query/distribute_agg_count.py index ebca81545c..67f7e28325 100644 --- a/tests/system-test/2-query/distribute_agg_count.py +++ b/tests/system-test/2-query/distribute_agg_count.py @@ -2,11 +2,11 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } @@ -25,7 +25,7 @@ class TDTestCase: same_sql = f"select sum(c) from (select {col_name} ,1 as c from {tbname} where {col_name} is not null) " tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -37,7 +37,7 @@ class TDTestCase: def prepare_datas_of_distribute(self): - + # prepate datas for 20 tables distributed at different vgroups tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") @@ -108,17 +108,17 @@ class TDTestCase: vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -129,14 +129,14 @@ class TDTestCase: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") def check_count_distribute_diff_vnode(self,col_name): - + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -145,13 +145,13 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - + max_sql = f"select count({col_name}) from stb1 where tbname in ({tbname_filters});" same_sql = f"select sum(c) from (select {col_name} ,1 as c from stb1 where tbname in ({tbname_filters}) and {col_name} is not null) " tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -162,8 +162,8 @@ class TDTestCase: tdLog.info(" count function work as expected, sql : %s "% max_sql) def check_count_status(self): - # check max function work status - + # check max function work status + tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] @@ -172,26 +172,26 @@ class TDTestCase: tdSql.query("desc stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_count_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): self.check_count_distribute_diff_vnode(colname) else: - # self.check_count_distribute_diff_vnode(colname) # bug for tag + # self.check_count_distribute_diff_vnode(colname) # bug for tag pass - + def distribute_agg_query(self): # basic filter tdSql.query("select count(c1) from stb1 ") @@ -212,12 +212,12 @@ class TDTestCase: tdSql.query("select count(c1) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all + # union all tdSql.query("select count(c1) from stb1 union all select count(c1) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,184) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") @@ -225,7 +225,7 @@ class TDTestCase: tdSql.execute(" create table tb1 using st tags(1) ") tdSql.execute(" create table tb2 using st tags(2) ") - + for i in range(10): ts = i*10 + self.ts tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") @@ -236,7 +236,7 @@ class TDTestCase: tdSql.checkData(0,0,10) tdSql.checkData(0,1,10) - # group by + # group by tdSql.execute(" use testdb ") tdSql.query(" select count(*) from stb1 ") @@ -251,7 +251,7 @@ class TDTestCase: # partition by tbname or partition by tag tdSql.query("select max(c1),tbname from stb1 partition by tbname") query_data = tdSql.queryResult - + for row in query_data: tbname = row[1] tdSql.query(" select max(c1) from %s "%tbname) @@ -259,7 +259,7 @@ class TDTestCase: tdSql.query("select max(c1),tbname from stb1 partition by t1") query_data = tdSql.queryResult - + for row in query_data: tbname = row[1] tdSql.query(" select max(c1) from %s "%tbname) @@ -287,7 +287,7 @@ class TDTestCase: self.check_count_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index c7e074095b..d4b71dbdd7 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -2,11 +2,11 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } @@ -25,7 +25,7 @@ class TDTestCase: same_sql = f"select {col_name} from {tbname} order by {col_name} desc limit 1" tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -37,7 +37,7 @@ class TDTestCase: def prepare_datas_of_distribute(self): - + # prepate datas for 20 tables distributed at different vgroups tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") @@ -108,17 +108,17 @@ class TDTestCase: vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -129,14 +129,14 @@ class TDTestCase: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") def check_max_distribute_diff_vnode(self,col_name): - + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -145,13 +145,13 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - + max_sql = f"select max({col_name}) from stb1 where tbname in ({tbname_filters});" same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) order by {col_name} desc limit 1" tdSql.query(max_sql) - max_result = tdSql.queryResult + max_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -162,8 +162,8 @@ class TDTestCase: tdLog.info(" max function work as expected, sql : %s "% max_sql) def check_max_status(self): - # check max function work status - + # check max function work status + tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] @@ -172,26 +172,26 @@ class TDTestCase: tdSql.query("desc stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_max_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): self.check_max_distribute_diff_vnode(colname) else: - # self.check_max_distribute_diff_vnode(colname) # bug for tag + # self.check_max_distribute_diff_vnode(colname) # bug for tag pass - + def distribute_agg_query(self): # basic filter tdSql.query("select max(c1) from stb1 where c1 is null") @@ -212,12 +212,12 @@ class TDTestCase: tdSql.query("select max(c1) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all + # union all tdSql.query("select max(c1) from stb1 union all select max(c1) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,28) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") @@ -225,7 +225,7 @@ class TDTestCase: tdSql.execute(" create table tb1 using st tags(1) ") tdSql.execute(" create table tb2 using st tags(2) ") - + for i in range(10): ts = i*10 + self.ts tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") @@ -236,7 +236,7 @@ class TDTestCase: tdSql.checkData(0,0,9) tdSql.checkData(0,0,9.00000) - # group by + # group by tdSql.execute(" use testdb ") tdSql.query(" select max(c1),c1 from stb1 group by t1 ") tdSql.checkRows(20) @@ -263,13 +263,13 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,28) tdSql.checkData(0,1,19) - tdSql.checkData(0,2,311110.000000000) - tdSql.checkData(0,3,2109) + tdSql.checkData(0,2,311110.000000000) + tdSql.checkData(0,3,2109) # partition by tbname or partition by tag tdSql.query("select max(c1),tbname from stb1 partition by tbname") query_data = tdSql.queryResult - + for row in query_data: tbname = row[1] tdSql.query(" select max(c1) from %s "%tbname) @@ -277,7 +277,7 @@ class TDTestCase: tdSql.query("select max(c1),tbname from stb1 partition by t1") query_data = tdSql.queryResult - + for row in query_data: tbname = row[1] tdSql.query(" select max(c1) from %s "%tbname) @@ -305,7 +305,7 @@ class TDTestCase: self.check_max_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index d8f93a01f5..059efe02cd 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -2,11 +2,11 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } @@ -25,7 +25,7 @@ class TDTestCase: same_sql = f"select {col_name} from {tbname} where {col_name} is not null order by {col_name} asc limit 1" tdSql.query(min_sql) - min_result = tdSql.queryResult + min_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -37,7 +37,7 @@ class TDTestCase: def prepare_datas_of_distribute(self): - + # prepate datas for 20 tables distributed at different vgroups tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") @@ -108,17 +108,17 @@ class TDTestCase: vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -129,14 +129,14 @@ class TDTestCase: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") def check_min_distribute_diff_vnode(self,col_name): - + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -145,13 +145,13 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - + min_sql = f"select min({col_name}) from stb1 where tbname in ({tbname_filters});" same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null order by {col_name} asc limit 1" tdSql.query(min_sql) - min_result = tdSql.queryResult + min_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -162,8 +162,8 @@ class TDTestCase: tdLog.info(" min function work as expected, sql : %s "% min_sql) def check_min_status(self): - # check max function work status - + # check max function work status + tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] @@ -172,26 +172,26 @@ class TDTestCase: tdSql.query("desc stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_min_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): self.check_min_distribute_diff_vnode(colname) else: - # self.check_min_distribute_diff_vnode(colname) # bug for tag + # self.check_min_distribute_diff_vnode(colname) # bug for tag pass - + def distribute_agg_query(self): # basic filter tdSql.query("select min(c1) from stb1 where c1 is null") @@ -212,12 +212,12 @@ class TDTestCase: tdSql.query("select min(c1) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all + # union all tdSql.query("select min(c1) from stb1 union all select min(c1) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,0) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") @@ -225,7 +225,7 @@ class TDTestCase: tdSql.execute(" create table tb1 using st tags(1) ") tdSql.execute(" create table tb2 using st tags(2) ") - + for i in range(10): ts = i*10 + self.ts tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") @@ -236,7 +236,7 @@ class TDTestCase: tdSql.checkData(0,0,0) tdSql.checkData(0,0,0.00000) - # group by + # group by tdSql.execute(" use testdb ") tdSql.query(" select min(c1),c1 from stb1 group by t1 ") tdSql.checkRows(20) @@ -261,12 +261,12 @@ class TDTestCase: tdSql.query("select min(c1),ceil(t1),pow(c2,1)+2,abs(t3) from stb1 where c1>12") tdSql.checkRows(1) tdSql.checkData(0,0,13) - tdSql.checkData(0,2,144445.000000000) - + tdSql.checkData(0,2,144445.000000000) + # partition by tbname or partition by tag tdSql.query("select min(c1),tbname from stb1 partition by tbname") query_data = tdSql.queryResult - + for row in query_data: tbname = row[1] tdSql.query(" select min(c1) from %s "%tbname) @@ -274,7 +274,7 @@ class TDTestCase: tdSql.query("select min(c1),tbname from stb1 partition by t1") query_data = tdSql.queryResult - + for row in query_data: tbname = row[1] tdSql.query(" select min(c1) from %s "%tbname) @@ -303,7 +303,7 @@ class TDTestCase: self.check_min_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_spread.py b/tests/system-test/2-query/distribute_agg_spread.py index 8d611007f3..842a74628d 100644 --- a/tests/system-test/2-query/distribute_agg_spread.py +++ b/tests/system-test/2-query/distribute_agg_spread.py @@ -2,11 +2,11 @@ from util.log import * from util.cases import * from util.sql import * import numpy as np -import random +import random class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } @@ -25,7 +25,7 @@ class TDTestCase: same_sql = f"select max({col_name})-min({col_name}) from {tbname}" tdSql.query(spread_sql) - spread_result = tdSql.queryResult + spread_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -37,7 +37,7 @@ class TDTestCase: def prepare_datas_of_distribute(self): - + # prepate datas for 20 tables distributed at different vgroups tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") @@ -108,17 +108,17 @@ class TDTestCase: vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -129,14 +129,14 @@ class TDTestCase: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") def check_spread_distribute_diff_vnode(self,col_name): - + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -145,13 +145,13 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - + spread_sql = f"select spread({col_name}) from stb1 where tbname in ({tbname_filters})" same_sql = f"select max({col_name}) - min({col_name}) from stb1 where tbname in ({tbname_filters})" tdSql.query(spread_sql) - spread_result = tdSql.queryResult + spread_result = tdSql.queryResult tdSql.query(same_sql) same_result = tdSql.queryResult @@ -162,8 +162,8 @@ class TDTestCase: tdLog.info(" spread function work as expected, sql : %s "% spread_sql) def check_spread_status(self): - # check max function work status - + # check max function work status + tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] @@ -172,26 +172,26 @@ class TDTestCase: tdSql.query("desc stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_spread_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): self.check_spread_distribute_diff_vnode(colname) else: - # self.check_spread_distribute_diff_vnode(colname) # bug for tag + # self.check_spread_distribute_diff_vnode(colname) # bug for tag pass - + def distribute_agg_query(self): # basic filter tdSql.query("select spread(c1) from stb1 where c1 is null") @@ -212,12 +212,12 @@ class TDTestCase: tdSql.query("select spread(c1) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all + # union all tdSql.query("select spread(c1) from stb1 union all select max(c1)-min(c1) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,28.000000000) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") @@ -225,7 +225,7 @@ class TDTestCase: tdSql.execute(" create table tb1 using st tags(1) ") tdSql.execute(" create table tb2 using st tags(2) ") - + for i in range(10): ts = i*10 + self.ts tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") @@ -236,7 +236,7 @@ class TDTestCase: tdSql.checkData(0,0,9.000000000) tdSql.checkData(0,0,9.00000) - # group by + # group by tdSql.execute(" use testdb ") tdSql.query(" select max(c1),c1 from stb1 group by t1 ") tdSql.checkRows(20) @@ -272,7 +272,7 @@ class TDTestCase: self.check_spread_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/distribute_agg_sum.py b/tests/system-test/2-query/distribute_agg_sum.py index d4e9dfb1fb..90d1edca90 100644 --- a/tests/system-test/2-query/distribute_agg_sum.py +++ b/tests/system-test/2-query/distribute_agg_sum.py @@ -7,7 +7,7 @@ import platform class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143, "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } @@ -24,7 +24,7 @@ class TDTestCase: sum_sql = f"select sum({col_name}) from {tbname};" same_sql = f"select {col_name} from {tbname} where {col_name} is not null " - + tdSql.query(same_sql) pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): @@ -35,7 +35,7 @@ class TDTestCase: tdSql.checkData(0,0,pre_sum) def prepare_datas_of_distribute(self): - + # prepate datas for 20 tables distributed at different vgroups tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") tdSql.execute(" use testdb ") @@ -106,17 +106,17 @@ class TDTestCase: vgroups = tdSql.queryResult vnode_tables={} - + for vgroup_id in vgroups: vnode_tables[vgroup_id[0]]=[] - + # check sub_table of per vnode ,make sure sub_table has been distributed tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] for table_name in table_names: - vnode_tables[table_name[6]].append(table_name[0]) + vnode_tables[table_name[6]].append(table_name[0]) self.vnode_disbutes = vnode_tables count = 0 @@ -127,14 +127,14 @@ class TDTestCase: tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") def check_sum_distribute_diff_vnode(self,col_name): - + vgroup_ids = [] for k ,v in self.vnode_disbutes.items(): if len(v)>=2: vgroup_ids.append(k) - + distribute_tbnames = [] - + for vgroup_id in vgroup_ids: vnode_tables = self.vnode_disbutes[vgroup_id] distribute_tbnames.append(random.sample(vnode_tables,1)[0]) @@ -143,7 +143,7 @@ class TDTestCase: tbname_ins += "'%s' ,"%tbname tbname_filters = tbname_ins[:-1] - + sum_sql = f"select sum({col_name}) from stb1 where tbname in ({tbname_filters});" same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null " @@ -158,8 +158,8 @@ class TDTestCase: tdSql.checkData(0,0,pre_sum) def check_sum_status(self): - # check max function work status - + # check max function work status + tdSql.query("show tables like 'ct%'") table_names = tdSql.queryResult tablenames = [] @@ -168,26 +168,26 @@ class TDTestCase: tdSql.query("desc stb1") col_names = tdSql.queryResult - + colnames = [] for col_name in col_names: if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: colnames.append(col_name[0]) - + for tablename in tablenames: for colname in colnames: self.check_sum_functions(tablename,colname) - # check max function for different vnode + # check max function for different vnode for colname in colnames: if colname.startswith("c"): self.check_sum_distribute_diff_vnode(colname) else: - # self.check_sum_distribute_diff_vnode(colname) # bug for tag + # self.check_sum_distribute_diff_vnode(colname) # bug for tag pass - + def distribute_agg_query(self): # basic filter tdSql.query(" select sum(c1) from stb1 ") @@ -211,7 +211,7 @@ class TDTestCase: tdSql.query("select sum(c1) from stb1 where t1> 4 partition by tbname") tdSql.checkRows(15) - # union all + # union all tdSql.query("select sum(c1) from stb1 union all select sum(c1) from stb1 ") tdSql.checkRows(2) tdSql.checkData(0,0,2592) @@ -220,7 +220,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0,0,5184) - # join + # join tdSql.execute(" create database if not exists db ") tdSql.execute(" use db ") @@ -228,7 +228,7 @@ class TDTestCase: tdSql.execute(" create table tb1 using st tags(1) ") tdSql.execute(" create table tb2 using st tags(2) ") - + for i in range(10): ts = i*10 + self.ts tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") @@ -239,7 +239,7 @@ class TDTestCase: tdSql.checkData(0,0,45) tdSql.checkData(0,1,45.000000000) - # group by + # group by tdSql.execute(" use testdb ") # partition by tbname or partition by tag @@ -269,7 +269,7 @@ class TDTestCase: self.check_sum_status() self.distribute_agg_query() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index e40920c06c..09a046d6ef 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -23,7 +23,7 @@ class TDTestCase: self.time_step = 1000 def insert_datas_and_check_irate(self ,tbnums , rownums , time_step ): - + tdLog.info(" prepare datas for auto check irate function ") tdSql.execute(" create database test ") @@ -48,7 +48,7 @@ class TDTestCase: c9 = "'nchar_val'" c10 = ts tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})") - + tdSql.execute("use test") tbnames = ["stb", "sub_tb_1"] support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] @@ -204,11 +204,11 @@ class TDTestCase: # used for sub table tdSql.query("select irate(abs(c1+c2)) from ct1") tdSql.checkData(0, 0, 0.000000000) - + # mix with common col tdSql.error("select c1, irate(c1) from ct1") - + # mix with common functions tdSql.error("select irate(c1), abs(c1) from ct4 ") @@ -236,7 +236,7 @@ class TDTestCase: "select irate(c1+c2)/10 from stb1 where c1 = 5 partition by tbname ") tdSql.checkRows(2) tdSql.checkData(0, 0, 0.000000000) - + def irate_Arithmetic(self): pass diff --git a/tests/system-test/2-query/last.py b/tests/system-test/2-query/last.py index d07d0c83eb..bae77b582c 100644 --- a/tests/system-test/2-query/last.py +++ b/tests/system-test/2-query/last.py @@ -222,9 +222,9 @@ class TDTestCase: if vgroups_num >= 2: tdLog.info(f'This scene with {vgroups_num} vgroups is ok!') continue - else: - tdLog.exit( - 'This scene does not meet the requirements with {vgroups_num} vgroup!\n') + # else: + # tdLog.exit( + # f'This scene does not meet the requirements with {vgroups_num} vgroup!\n') for i in range(self.tbnum): for j in range(self.rowNum): diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index f08a4b20de..b8e0aaf52e 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -10,13 +10,13 @@ from util.cases import * class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"udfDebugFlag":143} def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - + def prepare_datas(self): tdSql.execute( '''create table stb1 @@ -24,7 +24,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -69,12 +69,12 @@ class TDTestCase: def check_result_auto_log(self ,origin_query , log_query): - + log_result = tdSql.getResult(log_query) origin_result = tdSql.getResult(origin_query) auto_result =[] - + for row in origin_result: row_check = [] for elem in row: @@ -91,20 +91,20 @@ class TDTestCase: for row_index , row in enumerate(log_result): for col_index , elem in enumerate(row): if auto_result[row_index][col_index] != elem: - check_status = False + check_status = False if not check_status: tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) sys.exit(1) else: tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) - + def check_result_auto_log2(self ,origin_query , log_query): log_result = tdSql.getResult(log_query) origin_result = tdSql.getResult(origin_query) auto_result =[] - + for row in origin_result: row_check = [] for elem in row: @@ -121,7 +121,7 @@ class TDTestCase: for row_index , row in enumerate(log_result): for col_index , elem in enumerate(row): if auto_result[row_index][col_index] != elem: - check_status = False + check_status = False if not check_status: tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) sys.exit(1) @@ -133,7 +133,7 @@ class TDTestCase: origin_result = tdSql.getResult(origin_query) auto_result =[] - + for row in origin_result: row_check = [] for elem in row: @@ -150,7 +150,7 @@ class TDTestCase: for row_index , row in enumerate(log_result): for col_index , elem in enumerate(row): if auto_result[row_index][col_index] != elem: - check_status = False + check_status = False if not check_status: tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) sys.exit(1) @@ -161,7 +161,7 @@ class TDTestCase: origin_result = tdSql.getResult(origin_query) auto_result =[] - + for row in origin_result: row_check = [] for elem in row: @@ -178,13 +178,13 @@ class TDTestCase: for row_index , row in enumerate(log_result): for col_index , elem in enumerate(row): if auto_result[row_index][col_index] != elem: - check_status = False + check_status = False if not check_status: tdLog.notice("log function value has not as expected , sql is \"%s\" "%log_query ) sys.exit(1) else: tdLog.info("log value check pass , it work as expected ,sql is \"%s\" "%log_query ) - + def test_errors(self): error_sql_lists = [ "select log from t1", @@ -218,42 +218,42 @@ class TDTestCase: ] for error_sql in error_sql_lists: tdSql.error(error_sql) - + def support_types(self): type_error_sql_lists = [ - "select log(ts ,2 ) from t1" , + "select log(ts ,2 ) from t1" , "select log(c7,c2 ) from t1", "select log(c8,c1 ) from t1", "select log(c9,c2 ) from t1", - "select log(ts,c7 ) from ct1" , + "select log(ts,c7 ) from ct1" , "select log(c7,c9 ) from ct1", "select log(c8,c2 ) from ct1", "select log(c9,c1 ) from ct1", - "select log(ts,2 ) from ct3" , + "select log(ts,2 ) from ct3" , "select log(c7,2 ) from ct3", "select log(c8,2 ) from ct3", "select log(c9,2 ) from ct3", - "select log(ts,2 ) from ct4" , + "select log(ts,2 ) from ct4" , "select log(c7,2 ) from ct4", "select log(c8,2 ) from ct4", "select log(c9,2 ) from ct4", - "select log(ts,2 ) from stb1" , + "select log(ts,2 ) from stb1" , "select log(c7,2 ) from stb1", "select log(c8,2 ) from stb1", "select log(c9,2 ) from stb1" , - "select log(ts,2 ) from stbbb1" , + "select log(ts,2 ) from stbbb1" , "select log(c7,2 ) from stbbb1", "select log(ts,2 ) from tbname", "select log(c9,2 ) from tbname" ] - + for type_sql in type_error_sql_lists: tdSql.error(type_sql) - - + + type_sql_lists = [ "select log(c1,2 ) from t1", "select log(c2,2 ) from t1", @@ -283,16 +283,16 @@ class TDTestCase: "select log(c5,2 ) from stb1", "select log(c6,2 ) from stb1", - "select log(c6,2) as alisb from stb1", - "select log(c6,2) alisb from stb1", + "select log(c6,2) as alisb from stb1", + "select log(c6,2) alisb from stb1", ] for type_sql in type_sql_lists: tdSql.query(type_sql) - + def basic_log_function(self): - # basic query + # basic query tdSql.query("select c1 from ct3") tdSql.checkRows(0) tdSql.query("select c1 from t1") @@ -344,7 +344,7 @@ class TDTestCase: self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,2), log(c2 ,2) ,log(c3, 2), log(c4 ,2), log(c5 ,2) from t1") self.check_result_auto_log1( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,1), log(c2 ,1) ,log(c3, 1), log(c4 ,1), log(c5 ,1) from t1") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 from t1", "select log(c1 ,-10), log(c2 ,-10) ,log(c3, -10), log(c4 ,-10), log(c5 ,-10) from t1") - + # used for sub table tdSql.query("select c1 ,log(c1 ,3) from ct1") tdSql.checkData(0, 1, 1.892789261) @@ -382,18 +382,18 @@ class TDTestCase: tdSql.checkData(4 , 2 , None) tdSql.checkData(4 , 3 , None) - # # used for stable table - + # # used for stable table + tdSql.query("select log(c1, 2) from stb1") tdSql.checkRows(25) - + # used for not exists table tdSql.error("select log(c1, 2) from stbbb1") tdSql.error("select log(c1, 2) from tbname") tdSql.error("select log(c1, 2) from ct5") - # mix with common col + # mix with common col tdSql.query("select c1, log(c1 ,2) from ct1") tdSql.checkData(0 , 0 ,8) tdSql.checkData(0 , 1 ,3.000000000) @@ -418,7 +418,7 @@ class TDTestCase: tdSql.checkData(0 , 1 ,None) tdSql.checkData(0 , 2 ,None) tdSql.checkData(0 , 3 ,None) - + tdSql.checkData(3 , 0 , 6) tdSql.checkData(3 , 1 , 2.584962501) tdSql.checkData(3 , 2 ,6.66000) @@ -439,7 +439,7 @@ class TDTestCase: tdSql.query("select max(c5), count(c5) from stb1") tdSql.query("select max(c5), count(c5) from ct1") - + # bug fix for count tdSql.query("select count(c1) from ct4 ") tdSql.checkData(0,0,9) @@ -450,7 +450,7 @@ class TDTestCase: tdSql.query("select count(*) from stb1 ") tdSql.checkData(0,0,25) - # # bug fix for compute + # # bug fix for compute tdSql.query("select c1, log(c1 ,2) -0 ,log(c1-4 ,2)-0 from ct4 ") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) @@ -507,40 +507,40 @@ class TDTestCase: # base is an regular number ,int or double tdSql.query("select c1, log(c1, 2) from ct1") tdSql.checkData(0, 1,3.000000000) - tdSql.query("select c1, log(c1, 2.0) from ct1") + tdSql.query("select c1, log(c1, 2.0) from ct1") tdSql.checkData(0, 1, 3.000000000) - tdSql.query("select c1, log(1, 2.0) from ct1") + tdSql.query("select c1, log(1, 2.0) from ct1") tdSql.checkData(0, 1, 0.000000000) tdSql.checkRows(13) # # bug for compute in functions - # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.query("select c1, abs(1/0) from ct1") # tdSql.checkData(0, 0, 8) # tdSql.checkData(0, 1, 1) - tdSql.query("select c1, log(1, 2.0) from ct1") + tdSql.query("select c1, log(1, 2.0) from ct1") tdSql.checkData(0, 1, 0.000000000) tdSql.checkRows(13) # two cols start log(x,y) - tdSql.query("select c1,c2, log(c1,c2) from ct1") + tdSql.query("select c1,c2, log(c1,c2) from ct1") tdSql.checkData(0, 2, 0.182485070) tdSql.checkData(1, 2, 0.172791608) tdSql.checkData(4, 2, None) - tdSql.query("select c1,c2, log(c2,c1) from ct1") + tdSql.query("select c1,c2, log(c2,c1) from ct1") tdSql.checkData(0, 2, 5.479900349) tdSql.checkData(1, 2, 5.787318105) tdSql.checkData(4, 2, None) - tdSql.query("select c1, log(2.0 , c1) from ct1") + tdSql.query("select c1, log(2.0 , c1) from ct1") tdSql.checkData(0, 1, 0.333333333) tdSql.checkData(1, 1, 0.356207187) tdSql.checkData(4, 1, None) - tdSql.query("select c1, log(2.0 , ceil(abs(c1))) from ct1") + tdSql.query("select c1, log(2.0 , ceil(abs(c1))) from ct1") tdSql.checkData(0, 1, 0.333333333) tdSql.checkData(1, 1, 0.356207187) tdSql.checkData(4, 1, None) @@ -580,10 +580,10 @@ class TDTestCase: tdSql.checkData(0,3,8.000000000) tdSql.checkData(0,4,7.900000000) tdSql.checkData(0,5,3.000000000) - + def log_Arithmetic(self): pass - + def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") @@ -612,13 +612,13 @@ class TDTestCase: self.check_result_auto_log( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1), log(c2) ,log(c3), log(c4), log(c5) ,log(c6) from sub1_bound") self.check_result_auto_log2( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c4,2), log(c5,2) ,log(c6,2) from sub1_bound") self.check_result_auto_log__10( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select log(c1,-10), log(c2,-10) ,log(c3,-10), log(c4,-10), log(c5,-10) ,log(c6,-10) from sub1_bound") - + self.check_result_auto_log2( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1,2), log(c2,2) ,log(c3,2), log(c3,2), log(c2,2) ,log(c1,2) from sub1_bound") self.check_result_auto_log( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select log(c1), log(c2) ,log(c3), log(c3), log(c2) ,log(c1) from sub1_bound") self.check_result_auto_log2("select abs(abs(abs(abs(abs(abs(abs(abs(abs(c1))))))))) nest_col_func from sub1_bound" , "select log(abs(c1) ,2) from sub1_bound" ) - + # check basic elem for table per row tdSql.query("select log(abs(c1),2) ,log(abs(c2),2) , log(abs(c3),2) , log(abs(c4),2), log(abs(c5),2), log(abs(c6),2) from sub1_bound ") tdSql.checkData(0,0,math.log(2147483647,2)) @@ -683,45 +683,45 @@ class TDTestCase: self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) ,log(c5,2) from stb1 where c1 > 0 order by tbname" ) self.check_result_auto_log2( " select t1,c5 from stb1 where c1 > 0 order by tbname " , "select log(t1,2) , log(c5,2) from stb1 where c1 > 0 order by tbname" ) pass - + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() tdLog.printNoPrefix("==========step1:create table ==============") - + self.prepare_datas() - tdLog.printNoPrefix("==========step2:test errors ==============") + tdLog.printNoPrefix("==========step2:test errors ==============") self.test_errors() - - tdLog.printNoPrefix("==========step3:support types ============") + + tdLog.printNoPrefix("==========step3:support types ============") self.support_types() - tdLog.printNoPrefix("==========step4: log basic query ============") + tdLog.printNoPrefix("==========step4: log basic query ============") self.basic_log_function() - tdLog.printNoPrefix("==========step5: big number log query ============") + tdLog.printNoPrefix("==========step5: big number log query ============") self.test_big_number() - tdLog.printNoPrefix("==========step6: base number for log query ============") + tdLog.printNoPrefix("==========step6: base number for log query ============") self.log_base_test() - tdLog.printNoPrefix("==========step7: log boundary query ============") + tdLog.printNoPrefix("==========step7: log boundary query ============") self.check_boundary_values() - tdLog.printNoPrefix("==========step8: log filter query ============") + tdLog.printNoPrefix("==========step8: log filter query ============") self.abs_func_filter() tdLog.printNoPrefix("==========step9: check log result of stable query ============") - self.support_super_table_test() + self.support_super_table_test() def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py index c9df6f61bb..e0fb986d79 100644 --- a/tests/system-test/2-query/query_cols_tags_and_or.py +++ b/tests/system-test/2-query/query_cols_tags_and_or.py @@ -50,7 +50,7 @@ class TDTestCase: tb_name = tdCom.getLongName(8, "letters") tdSql.execute( f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 float, t6 double, t7 binary(100), t8 nchar(200), t9 bool, t10 tinyint unsigned, t11 smallint unsigned, t12 int unsigned, t13 bigint unsigned)") - for i in range(1, count+1): + for i in range(1, count+1): tdSql.execute( f'CREATE TABLE {tb_name}_sub_{i} using {tb_name} tags ({i}, {i}, {i}, {i}, {i}.{i}, {i}.{i}, "binary{i}", "nchar{i}", true, {i}, {i}, {i}, {i})') self.insertData(f'{tb_name}_sub_{i}') @@ -412,7 +412,7 @@ class TDTestCase: query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c1 > 2 and c1 >= 3 or c1 < 1 or c1 <= 0 or c1 =2 or c1 != 1 or c1 <> 1 and c1 is null or c1 between 2 and 3 and c1 not between 1 and 1 and c1 in (2, 3) and c1 not in (1, 2)' res = tdSql.query(query_sql) tdSql.checkRows(1) - + def queryUtinyintCol(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm # > @@ -497,7 +497,7 @@ class TDTestCase: query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c10 > 2 and c10 >= 3 or c10 < 1 or c10 <= 0 or c10 =2 or c10 != 1 or c10 <> 1 and c10 is null or c10 between 2 and 3 and c10 not between 1 and 1 and c10 in (2, 3) and c10 not in (1, 2)' res = tdSql.query(query_sql) tdSql.checkRows(10) - + def querySmallintCol(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm # > @@ -582,7 +582,7 @@ class TDTestCase: query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c2 > 0 and c2 >= 1 or c2 < 4 and c2 <= 3 and c2 != 1 and c2 <> 2 and c2 = 3 or c2 is not null and c2 between 2 and 3 and c2 not between 1 and 2 and c2 in (2,3) and c2 not in (1,2)' tdSql.query(query_sql) tdSql.checkRows(11) - + def queryUsmallintCol(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm # > @@ -752,7 +752,7 @@ class TDTestCase: query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c3 > 0 and c3 >= 1 or c3 < 5 and c3 <= 4 and c3 != 2 and c3 <> 2 and c3 = 4 or c3 is not null and c3 between 2 and 4 and c3 not between 1 and 2 and c3 in (2,4) and c3 not in (1,2)' tdSql.query(query_sql) tdSql.checkRows(11) - + def queryUintCol(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm # > @@ -1086,7 +1086,7 @@ class TDTestCase: query_sql = f'select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13 from {tb_name} where c5 > 0 and c5 >= 1 or c5 < 5 and c5 <= 6.6 and c5 != 2 and c5 <> 2 and c5 = 4 or c5 is not null and c5 between 2 and 4 and c5 not between 1 and 2 and c5 in (2,4) and c5 not in (1,2)' tdSql.query(query_sql) tdSql.checkRows(11) - + def queryDoubleCol(self, tb_name, check_elm=None): select_elm = "*" if check_elm is None else check_elm # > @@ -1711,19 +1711,19 @@ class TDTestCase: tdSql.checkRows(4) tdSql.checkEqual(self.queryLastC10(query_sql), 7) - ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F + ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 1 and c3 <4) or (c3 >= 4 or c7 is not Null) and c9 <> true' tdSql.query(query_sql) tdSql.checkRows(3) tdSql.checkEqual(self.queryLastC10(query_sql), 10) - ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F + ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F query_sql = f'select * from {tb_name} where (c1 != 1 or (c2 <= 2 and c3 >= 4) or (c3 >= 4 or c7 is not Null)) and c9 != false' tdSql.query(query_sql) tdSql.checkRows(9) tdSql.checkEqual(self.queryLastC10(query_sql), 9) - ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) + ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 3 and c3 > 4) and c3 <= 5 and (c7 is not Null and c9 != false)' tdSql.query(query_sql) tdSql.checkRows(2) @@ -1780,17 +1780,17 @@ class TDTestCase: tdSql.query(query_sql) tdSql.checkRows(55) - ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F + ## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F query_sql = f'select * from {tb_name} where t1 != 1 or (t2 <= 1 and t3 <4) or (t3 >= 4 or t7 is not Null) and t9 <> true' tdSql.query(query_sql) tdSql.checkRows(55) - ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F + ## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F query_sql = f'select * from {tb_name} where (t1 != 1 or (t2 <= 2 and t3 >= 4) or (t3 >= 4 or t7 is not Null)) and t9 != false' tdSql.query(query_sql) tdSql.checkRows(55) - ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) + ## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G) query_sql = f'select * from {tb_name} where t1 != 1 or (t2 <= 3 and t3 > 4) and t3 <= 5 and (t7 is not Null and t9 != false)' tdSql.query(query_sql) tdSql.checkRows(44) @@ -2033,7 +2033,7 @@ class TDTestCase: self.checkColType(tb_name, check_elm) else: self.checkColType(stb_name, check_elm) - + def checkStbTagTypeOperator(self): ''' Super table full tag type and operator @@ -2089,7 +2089,7 @@ class TDTestCase: tb_name = self.initStb() self.queryColPreCal(f'{tb_name}_sub_1') self.queryTagPreCal(tb_name) - + def checkMultiTb(self): ''' test "or" in multi ordinary table @@ -2110,7 +2110,7 @@ class TDTestCase: ''' tb_name = self.initStb() self.queryMultiTbWithTag(tb_name) - + def checkMultiStbJoin(self): ''' join test From 5a4fd268f2e4a9841c654ef5a30f4af5fd47c1ee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:21:21 +0800 Subject: [PATCH 15/36] other: merge 3.0 --- source/os/src/osSystem.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 9c66ac1df8..ad7fa57182 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -18,10 +18,6 @@ #include "os.h" #if defined(WINDOWS) -BOOL WINAPI CtrlHandler(DWORD fdwCtrlType) { - printf("\n" TAOS_CONSOLE_PROMPT_HEADER); - return TRUE; -} #elif defined(_TD_DARWIN_64) #else #include @@ -128,7 +124,6 @@ int taosSetConsoleEcho(bool on) { void taosSetTerminalMode() { #if defined(WINDOWS) - SetConsoleCtrlHandler(CtrlHandler, TRUE); #else struct termios newtio; @@ -179,7 +174,6 @@ int32_t taosGetOldTerminalMode() { void taosResetTerminalMode() { #if defined(WINDOWS) - SetConsoleCtrlHandler(CtrlHandler, FALSE); #else if (tcsetattr(0, TCSANOW, &oldtio) != 0) { fprintf(stderr, "Fail to reset the terminal properties!\n"); From 479639d1fc679d0b7fbdd65b9659891837525fb5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:21:55 +0800 Subject: [PATCH 16/36] other: merge 3.0 --- tools/CMakeLists.txt | 4 ++-- tools/shell/src/shellArguments.c | 3 +++ tools/shell/src/shellEngine.c | 24 +++++++++++++----------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index e6cd47b91d..995a66acd8 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -43,7 +43,7 @@ IF (TD_WEBSOCKET) ENDIF() ENDIF () -IF (TD_TAOS_TOOLS) +#IF (TD_TAOS_TOOLS) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/tools/taos_tools/deps/avro/lang/c/src) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/client) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/common) @@ -51,7 +51,7 @@ IF (TD_TAOS_TOOLS) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/os) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/libs/transport) ADD_SUBDIRECTORY(taos-tools) -ENDIF () +#ENDIF () add_subdirectory(shell) IF (TD_BUILD_HTTP) diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index cdbdc0de60..466aa52390 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -19,6 +19,9 @@ #include "shellInt.h" +#define TAOS_CONSOLE_PROMPT_HEADER "taos> " +#define TAOS_CONSOLE_PROMPT_CONTINUE " -> " + #define SHELL_HOST "The auth string to use when connecting to the server." #define SHELL_PORT "The TCP/IP port number to use for the connection." #define SHELL_USER "The user name to use when connecting to the server." diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index eefb0aa8b2..56bc1ed6cc 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -41,7 +41,7 @@ static void shellPrintError(TAOS_RES *tres, int64_t st); static bool shellIsCommentLine(char *line); static void shellSourceFile(const char *file); static void shellGetGrantInfo(); -static void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context); + static void shellCleanup(void *arg); static void *shellCancelHandler(void *arg); static void *shellThreadLoop(void *arg); @@ -919,11 +919,14 @@ void shellGetGrantInfo() { fprintf(stdout, "\r\n"); } -void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&shell.cancelSem); } - -void shellSigintHandler(int32_t signum, void *sigInfo, void *context) { - // do nothing +#ifdef WINDOWS +BOOL shellQueryInterruptHandler(DWORD fdwCtrlType) { + tsem_post(&shell.cancelSem); + return TRUE; } +#else +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&shell.cancelSem); } +#endif void shellCleanup(void *arg) { taosResetTerminalMode(); } @@ -934,11 +937,10 @@ void *shellCancelHandler(void *arg) { taosMsleep(10); continue; } - - taosResetTerminalMode(); - printf("\r\nReceive SIGTERM or other signal, quit shell.\r\n"); - shellWriteHistory(); - shellExit(); + taos_kill_query(shell.conn); + #ifdef WINDOWS + printf("\n%s", shell.info.promptHeader); + #endif } return NULL; @@ -1022,7 +1024,7 @@ int32_t shellExecute() { taosSetSignal(SIGHUP, shellQueryInterruptHandler); taosSetSignal(SIGABRT, shellQueryInterruptHandler); - taosSetSignal(SIGINT, shellSigintHandler); + taosSetSignal(SIGINT, shellQueryInterruptHandler); shellGetGrantInfo(); From 8517e3a345ac08f47ee018e6c4f016184a40c5cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:23:05 +0800 Subject: [PATCH 17/36] other: merge 3.0 --- tests/system-test/7-tmq/tmqCommon.py | 35 +++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 164a6b24ba..a56f79d20f 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -20,6 +20,8 @@ import threading import requests import time # import socketfrom +import json +import toml import taos from util.log import * @@ -207,7 +209,7 @@ class TMQCom: def drop_ctable(self, tsql, dbname=None, count=1, default_ctbname_prefix="ctb",ctbStartIdx=0): for _ in range(count): - create_ctable_sql = f'drop table {dbname}.{default_ctbname_prefix}{ctbStartIdx};' + create_ctable_sql = f'drop table if exists {dbname}.{default_ctbname_prefix}{ctbStartIdx};' ctbStartIdx += 1 tdLog.info("drop ctb sql: %s"%create_ctable_sql) tsql.execute(create_ctable_sql) @@ -503,6 +505,37 @@ class TMQCom: break return + def create_ntable(self, tsql, dbname=None, tbname_prefix="ntb", tbname_index_start_num = 1, column_elm_list=None, colPrefix='c', tblNum=1, **kwargs): + tb_params = "" + if len(kwargs) > 0: + for param, value in kwargs.items(): + tb_params += f'{param} "{value}" ' + column_type_str = tdCom.gen_column_type_str(colPrefix, column_elm_list) + + for _ in range(tblNum): + create_table_sql = f'create table {dbname}.{tbname_prefix}{tbname_index_start_num} ({column_type_str}) {tb_params};' + tbname_index_start_num += 1 + tsql.execute(create_table_sql) + + def insert_rows_into_ntbl(self, tsql, dbname=None, tbname_prefix="ntb", tbname_index_start_num = 1, column_ele_list=None, startTs=None, tblNum=1, rows=1): + if startTs is None: + startTs = tdCom.genTs()[0] + + for tblIdx in range(tblNum): + for rowIdx in range(rows): + column_value_list = tdCom.gen_column_value_list(column_ele_list, f'{startTs}+{rowIdx}s') + column_value_str = '' + idx = 0 + for column_value in column_value_list: + if isinstance(column_value, str) and idx != 0: + column_value_str += f'"{column_value}", ' + else: + column_value_str += f'{column_value}, ' + idx += 1 + column_value_str = column_value_str.rstrip()[:-1] + insert_sql = f'insert into {dbname}.{tbname_prefix}{tblIdx+tbname_index_start_num} values ({column_value_str});' + tsql.execute(insert_sql) + def close(self): self.cursor.close() From c6490cbabeb763f8c76df91508a717a83de66ea6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:23:33 +0800 Subject: [PATCH 18/36] other: merge 3.0 --- tests/system-test/7-tmq/tmqDropNtb.py | 237 ++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 tests/system-test/7-tmq/tmqDropNtb.py diff --git a/tests/system-test/7-tmq/tmqDropNtb.py b/tests/system-test/7-tmq/tmqDropNtb.py new file mode 100644 index 0000000000..9200200588 --- /dev/null +++ b/tests/system-test/7-tmq/tmqDropNtb.py @@ -0,0 +1,237 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def __init__(self): + self.snapshot = 0 + self.vgroups = 4 + self.ctbNum = 100 + self.rowsPerTbl = 10 + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def waitSubscriptionExit(self, max_wait_count=20): + wait_cnt = 0 + while (wait_cnt < max_wait_count): + tdSql.query("show subscriptions") + if tdSql.getRows() == 0: + break + else: + time.sleep(1) + wait_cnt += 1 + + tdLog.info("wait subscriptions exit for %d s"%wait_cnt) + + # drop some ntbs + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ntb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 5, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdLog.info("start create database....") + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("start create normal tables....") + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 0 + elif self.snapshot == 1: + consumerId = 1 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop some ntables") + # drop 1/4 ctbls from half offset + paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) + paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) + tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows >= expectrowcnt * 3/4) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + self.waitSubscriptionExit() + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 1 end ...... ") + + + + # drop some ntbs and create some new ntbs + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'dbt', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], + 'ctbPrefix': 'ntb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'endTs': 0, + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1, + 'snapshot': 0} + paraDict['snapshot'] = self.snapshot + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tmqCom.initConsumerTable() + tdLog.info("start create database....") + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict["vgroups"],replica=1) + tdLog.info("start create normal tables....") + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("create topics from database") + topicFromDb = 'topic_dbt' + tdSql.execute("create topic %s as database %s" %(topicFromDb, paraDict['dbName'])) + + if self.snapshot == 0: + consumerId = 2 + elif self.snapshot == 1: + consumerId = 3 + + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2) + topicList = topicFromDb + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1,\ + enable.auto.commit:true,\ + auto.commit.interval.ms:1000,\ + auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop some ntables") + # drop 1/4 ctbls from half offset + paraDict["ctbStartIdx"] = paraDict["ctbStartIdx"] + int(paraDict["ctbNum"] * 1 / 2) + paraDict["ctbNum"] = int(paraDict["ctbNum"] / 4) + tmqCom.drop_ctable(tdSql, dbname=paraDict['dbName'], count=paraDict["ctbNum"], default_ctbname_prefix=paraDict["ctbPrefix"], ctbStartIdx=paraDict["ctbStartIdx"]) + + tdLog.info("start create some new normal tables....") + paraDict["ctbPrefix"] = 'newCtb' + paraDict["ctbNum"] = self.ctbNum + tmqCom.create_ntable(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_elm_list=paraDict["colSchema"], colPrefix='c', tblNum=paraDict["ctbNum"]) + tdLog.info("start insert data into these new normal tables....") + tmqCom.insert_rows_into_ntbl(tsql=tdSql, dbname=paraDict["dbName"], tbname_prefix=paraDict["ctbPrefix"], tbname_index_start_num = 1, column_ele_list=paraDict["colSchema"],startTs=paraDict["startTs"], tblNum=paraDict["ctbNum"], rows=paraDict["rowsPerTbl"]) + + tdLog.info("start to check consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + + if not ((totalConsumeRows >= expectrowcnt / 2 * (1 + 3/4)) and (totalConsumeRows < expectrowcnt)): + tdLog.exit("tmq consume rows error with snapshot = 0!") + + tdLog.info("wait subscriptions exit ....") + self.waitSubscriptionExit() + + tdSql.query("drop topic %s"%topicFromDb) + tdLog.info("success dorp topic: %s"%topicFromDb) + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + tdLog.printNoPrefix("=============================================") + tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + self.snapshot = 0 + # self.tmqCase1() + self.tmqCase2() + + tdLog.printNoPrefix("====================================================================") + tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") + self.snapshot = 1 + # self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 819bf03cd1ae8ed4308ec144e9d3aedc6e5e291e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:23:56 +0800 Subject: [PATCH 19/36] other: merge 3.0 --- tests/system-test/7-tmq/TD-17699.py | 129 ++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 tests/system-test/7-tmq/TD-17699.py diff --git a/tests/system-test/7-tmq/TD-17699.py b/tests/system-test/7-tmq/TD-17699.py new file mode 100644 index 0000000000..87b11f6f83 --- /dev/null +++ b/tests/system-test/7-tmq/TD-17699.py @@ -0,0 +1,129 @@ +import sys +import time +import socket +import os +import threading + +import taos +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 2, + 'stbName': 'stb0', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':16, 'count':1}, {'type': 'timestamp','count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 1000, + 'batchNum': 1000, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 20, + 'showMsg': 1, + 'showRow': 1} + + cdbName = 'cdb' + # some parameter to consumer processor + consumerId = 0 + expectrowcnt = 0 + topicList = '' + ifcheckdata = 0 + ifManualCommit = 1 + groupId = 'group.id:cgrp1' + autoCommit = 'enable.auto.commit:false' + autoCommitInterval = 'auto.commit.interval.ms:1000' + autoOffset = 'auto.offset.reset:earliest' + + pollDelay = 20 + showMsg = 1 + showRow = 1 + + hostname = socket.gethostname() + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + logSql = False + tdSql.init(conn.cursor(), logSql) + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + tdLog.info("step 1: create database, stb, ctb and insert data") + + tmqCom.initConsumerTable(self.cdbName) + + tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) + + self.paraDict["stbName"] = 'stb1' + tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) + tdCom.create_ctable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],tag_elm_list=self.paraDict['tagSchema'],count=self.paraDict["ctbNum"],default_ctbname_prefix=self.paraDict["ctbPrefix"]) + tmqCom.insert_data_2(tdSql,self.paraDict["dbName"],self.paraDict["ctbPrefix"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"],self.paraDict["startTs"],self.paraDict["ctbStartIdx"]) + # pThread1 = tmqCom.asyncInsertData(paraDict=self.paraDict) + + self.paraDict["stbName"] = 'stb2' + self.paraDict["ctbPrefix"] = 'newctb' + self.paraDict["batchNum"] = 1000 + tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) + tdCom.create_ctable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],tag_elm_list=self.paraDict['tagSchema'],count=self.paraDict["ctbNum"],default_ctbname_prefix=self.paraDict["ctbPrefix"]) + # tmqCom.insert_data_2(tdSql,self.paraDict["dbName"],self.paraDict["ctbPrefix"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"],self.paraDict["startTs"],self.paraDict["ctbStartIdx"]) + pThread2 = tmqCom.asyncInsertData(paraDict=self.paraDict) + + tdLog.info("create topics from db") + topicName1 = 'UpperCasetopic_%s'%(self.paraDict['dbName']) + tdSql.execute("create topic %s as database %s" %(topicName1, self.paraDict['dbName'])) + + topicList = topicName1 + ',' +topicName1 + keyList = '%s,%s,%s,%s'%(self.groupId,self.autoCommit,self.autoCommitInterval,self.autoOffset) + self.expectrowcnt = self.paraDict["rowsPerTbl"] * self.paraDict["ctbNum"] * 2 + tmqCom.insertConsumerInfo(self.consumerId, self.expectrowcnt,topicList,keyList,self.ifcheckdata,self.ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName) + + tmqCom.getStartConsumeNotifyFromTmqsim() + tdLog.info("drop one stable") + self.paraDict["stbName"] = 'stb1' + tdSql.execute("drop table %s.%s" %(self.paraDict['dbName'], self.paraDict['stbName'])) + tmqCom.drop_ctable(tdSql, dbname=self.paraDict['dbName'], count=self.paraDict["ctbNum"], default_ctbname_prefix=self.paraDict["ctbPrefix"]) + + # pThread2.join() + + tdLog.info("wait result from consumer, then check it") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if not (totalConsumeRows >= self.expectrowcnt/2 and totalConsumeRows <= self.expectrowcnt): + tdLog.info("act consume rows: %d, expect consume rows: between %d and %d"%(totalConsumeRows, self.expectrowcnt/2, self.expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + time.sleep(10) + tdSql.query("drop topic %s"%topicName1) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 2b199e7a1795803105eb53f48a9441111a03e973 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 21:24:10 +0800 Subject: [PATCH 20/36] other: merge 3.0 --- tests/system-test/fulltest.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index d3dd93f9ca..ec70d9ddbf 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -32,6 +32,9 @@ python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py +python3 ./test.py -f 1-insert/update_data_muti_rows.py + + python3 ./test.py -f 2-query/abs.py python3 ./test.py -f 2-query/abs.py -R python3 ./test.py -f 2-query/and_or_for_byte.py @@ -59,7 +62,8 @@ python3 ./test.py -f 2-query/char_length.py -R python3 ./test.py -f 2-query/check_tsdb.py python3 ./test.py -f 2-query/check_tsdb.py -R -# jira python3 ./test.py -f 1-insert/update_data.py +# python3 ./test.py -f 1-insert/update_data.py + python3 ./test.py -f 1-insert/delete_data.py python3 ./test.py -f 2-query/db.py From 4d4a52245ddb8548392b2922f9d5d5f5a0389c76 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Jul 2022 23:52:48 +0800 Subject: [PATCH 21/36] fix(query): update the time range after filter data block. --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 24 ++++++-- source/libs/executor/src/groupoperator.c | 2 +- source/libs/executor/src/joinoperator.c | 2 +- source/libs/executor/src/scanoperator.c | 58 ++----------------- source/libs/executor/src/sortoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 12 ++-- 7 files changed, 35 insertions(+), 67 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c6ff913837..94a4294d4a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -807,7 +807,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scan int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 012d25a0c9..eef292f3f2 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1333,7 +1333,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowRes, bool keep); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo) { if (pFilterNode == NULL || pBlock->info.rows == 0) { return; } @@ -1354,6 +1354,20 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { filterFreeInfo(filter); extractQualifiedTupleByFilterResult(pBlock, rowRes, keep); + + if (pColMatchInfo != NULL) { + for(int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) { + SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, i); + if (pInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->targetSlotId); + if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + blockDataUpdateTsWindow(pBlock, pInfo->targetSlotId); + break; + } + } + } + } + taosMemoryFree(rowRes); } @@ -3040,7 +3054,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, pInfo, &pAggInfo->groupResInfo, pAggInfo->aggSup.pResultBuf); - doFilter(pAggInfo->pCondition, pInfo->pRes); + doFilter(pAggInfo->pCondition, pInfo->pRes, NULL); if (!hasDataInGroupInfo(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -3401,7 +3415,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // do apply filter SSDataBlock* p = pProjectInfo->mergeDataBlocks ? pFinalRes : pRes; - doFilter(pProjectInfo->pFilterNode, p); + doFilter(pProjectInfo->pFilterNode, p, NULL); if (p->info.rows > 0) { break; } @@ -3543,7 +3557,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { break; } - doFilter(pInfo->pCondition, fillResult); + doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo); if (fillResult->info.rows > 0) { break; } @@ -4005,7 +4019,7 @@ static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) { } } - doFilter(pIndefInfo->pCondition, pInfo->pRes); + doFilter(pIndefInfo->pCondition, pInfo->pRes, NULL); size_t rows = pInfo->pRes->info.rows; if (rows > 0 || pOperator->status == OP_EXEC_DONE) { break; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 20630fd6ff..c83206b730 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -299,7 +299,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; while(1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes); + doFilter(pInfo->pCondition, pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 497de8347c..2e6c9bd351 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -211,7 +211,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { break; } if (pJoinInfo->pCondAfterMerge != NULL) { - doFilter(pJoinInfo->pCondAfterMerge, pRes); + doFilter(pJoinInfo->pCondAfterMerge, pRes, NULL); } if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9ecf6b96d7..31c20ca47b 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -265,7 +265,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca } int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); int64_t et = taosGetTimestampMs(); pTableScanInfo->readRecorder.filterTime += (et - st); @@ -274,6 +274,8 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->filterOutBlocks += 1; qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + } else { + qDebug("%s data block filter out, elapsed time:%"PRId64, GET_TASKID(pTaskInfo), (et - st)); } return TSDB_CODE_SUCCESS; @@ -1225,7 +1227,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock } } - doFilter(pInfo->pCondition, pInfo->pRes); + doFilter(pInfo->pCondition, pInfo->pRes, NULL); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); return 0; @@ -1761,55 +1763,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } - doFilter(pInfo->pCondition, pInfo->pRes); -#if 0 - SFilterInfo* filter = NULL; - - int32_t code = filterInitFromNode(pInfo->pCondition, &filter, 0); - - SFilterColumnParam param1 = {.numOfCols = pInfo->pRes->info.numOfCols, .pDataBlock = pInfo->pRes->pDataBlock}; - code = filterSetDataFromSlotId(filter, ¶m1); - - int8_t* rowRes = NULL; - bool keep = filterExecute(filter, pInfo->pRes, &rowRes, NULL, param1.numOfCols); - filterFreeInfo(filter); - - SSDataBlock* px = createOneDataBlock(pInfo->pRes, false); - blockDataEnsureCapacity(px, pInfo->pRes->info.rows); - - // TODO refactor - int32_t numOfRow = 0; - for (int32_t i = 0; i < pInfo->pRes->info.numOfCols; ++i) { - SColumnInfoData* pDest = taosArrayGet(px->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); - - if (keep) { - colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info); - numOfRow = pInfo->pRes->info.rows; - } else if (NULL != rowRes) { - numOfRow = 0; - for (int32_t j = 0; j < pInfo->pRes->info.rows; ++j) { - if (rowRes[j] == 0) { - continue; - } - - if (colDataIsNull_s(pSrc, j)) { - colDataAppendNULL(pDest, numOfRow); - } else { - colDataAppend(pDest, numOfRow, colDataGetData(pSrc, j), false); - } - - numOfRow += 1; - } - } else { - numOfRow = 0; - } - } - - px->info.rows = numOfRow; - pInfo->pRes = px; -#endif - + doFilter(pInfo->pCondition, pInfo->pRes, NULL); return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } @@ -2777,7 +2731,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc } int64_t st = taosGetTimestampMs(); - doFilter(pTableScanInfo->pFilterNode, pBlock); + doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo); int64_t et = taosGetTimestampMs(); pTableScanInfo->readRecorder.filterTime += (et - st); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index f019ee94b8..26dd772292 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -216,7 +216,7 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) { return NULL; } - doFilter(pInfo->pCondition, pBlock); + doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo); if (blockDataGetNumOfRows(pBlock) == 0) { continue; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b5966fc463..7c8b0ef649 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1178,7 +1178,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -1219,7 +1219,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -1256,7 +1256,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBlock); + doFilter(pInfo->pCondition, pBlock, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -1969,7 +1969,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_RES_TO_RETURN) { while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -2013,7 +2013,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pBInfo->pRes); + doFilter(pInfo->pCondition, pBInfo->pRes, NULL); bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); if (!hasRemain) { @@ -4626,7 +4626,7 @@ static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true); doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); - doFilter(miaInfo->pCondition, pRes); + doFilter(miaInfo->pCondition, pRes, NULL); if (pRes->info.rows >= pOperator->resultInfo.capacity) { break; } From 3f7a4a99fccf47354c5cda5643c09a2f3630825d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 00:05:48 +0800 Subject: [PATCH 22/36] other: update cmake. --- tools/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 995a66acd8..e6cd47b91d 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -43,7 +43,7 @@ IF (TD_WEBSOCKET) ENDIF() ENDIF () -#IF (TD_TAOS_TOOLS) +IF (TD_TAOS_TOOLS) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/tools/taos_tools/deps/avro/lang/c/src) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/client) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/common) @@ -51,7 +51,7 @@ ENDIF () INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/os) INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/include/libs/transport) ADD_SUBDIRECTORY(taos-tools) -#ENDIF () +ENDIF () add_subdirectory(shell) IF (TD_BUILD_HTTP) From e9936db95e832a41f7faa7c952c1cd21b831fa95 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 09:37:23 +0800 Subject: [PATCH 23/36] other: merge 3.0 --- include/libs/executor/executor.h | 2 +- source/libs/executor/inc/executil.h | 4 ++-- source/libs/executor/src/executil.c | 6 +++--- source/libs/executor/src/executor.c | 14 +++++--------- source/libs/executor/src/executorMain.c | 0 source/libs/executor/src/executorimpl.c | 2 +- 6 files changed, 12 insertions(+), 16 deletions(-) delete mode 100644 source/libs/executor/src/executorMain.c diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index bd8e366b15..65e20336cc 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -65,7 +65,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); * @return */ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, - SSchemaWrapper** pSchemaWrapper); + SSchemaWrapper** pSchema); /** * Set the input data block for the stream scan. diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 62ce56ee91..23732a6f9a 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -108,7 +108,7 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); -int32_t getGroupIdFromTableTags(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); size_t getTableTagsBufLen(const SNodeList* pGroups); SArray* createSortInfo(SNodeList* pNodeList); @@ -132,6 +132,6 @@ int32_t convertFillType(int32_t mode); int32_t resultrowComparAsc(const void* p1, const void* p2); -int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 1a34635fe8..a76253ab20 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -265,7 +265,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -int32_t isTableOk(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { +int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, bool* pQualified) { int32_t code = TSDB_CODE_SUCCESS; SMetaReader mr = {0}; @@ -356,7 +356,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); bool qualified = true; - code = isTableOk(info, pTagCond, metaHandle, &qualified); + code = isQualifiedTable(info, pTagCond, metaHandle, &qualified); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -392,7 +392,7 @@ size_t getTableTagsBufLen(const SNodeList* pGroups) { return keyLen; } -int32_t getGroupIdFromTableTags(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { +int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId) { SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); metaGetTableEntryByUid(&mr, uid); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7d3f16aedf..d8cd76d31e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -131,8 +131,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO return code; } -qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, - SSchemaWrapper** pSchemaWrapper) { +qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema) { if (msg == NULL) { // TODO create raw scan return NULL; @@ -166,7 +165,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n } } - *pSchemaWrapper = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw); + *pSchema = tCloneSSchemaWrapper(((SExecTaskInfo*)pTaskInfo)->schemaInfo.qsw); return pTaskInfo; } @@ -219,7 +218,7 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S if (pScanInfo->pTagCond != NULL) { bool qualified = false; STableKeyInfo info = {.groupId = 0, .uid = mr.me.uid}; - code = isTableOk(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); + code = isQualifiedTable(&info, pScanInfo->pTagCond, pScanInfo->readHandle.meta, &qualified); if (code != TSDB_CODE_SUCCESS) { qError("failed to filter new table, uid:0x%" PRIx64 ", %s", info.uid, idstr); continue; @@ -273,7 +272,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo STableKeyInfo keyInfo = {.uid = *uid, .groupId = 0}; if (bufLen > 0) { - code = getGroupIdFromTableTags(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, + code = getGroupIdFromTagsVal(pScanInfo->readHandle.meta, keyInfo.uid, pScanInfo->pGroupTags, keyBuf, &keyInfo.groupId); if (code != TSDB_CODE_SUCCESS) { return code; @@ -458,13 +457,11 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { int32_t qKillTask(qTaskInfo_t qinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo; - if (pTaskInfo == NULL) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - qDebug("%s execTask killed", GET_TASKID(pTaskInfo)); - setTaskKilled(pTaskInfo); + qAsyncKillTask(qinfo); // Wait for the query executing thread being stopped/ // Once the query is stopped, the owner of qHandle will be cleared immediately. @@ -484,7 +481,6 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) { qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); setTaskKilled(pTaskInfo); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 484cfda7bb..8162bb609d 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4362,7 +4362,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t groupNum = 0; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - int32_t code = getGroupIdFromTableTags(pHandle->meta, info->uid, group, keyBuf, &info->groupId); + int32_t code = getGroupIdFromTagsVal(pHandle->meta, info->uid, group, keyBuf, &info->groupId); if (code != TSDB_CODE_SUCCESS) { return code; } From a3b5f18997497a83e178761957ab639af42705d0 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 23 Jul 2022 09:58:23 +0800 Subject: [PATCH 24/36] test: add case for fix --- tests/system-test/7-tmq/tmqDnodeRestart.py | 27 +++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index e69f5b1eeb..9c4ba2be55 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -18,7 +18,7 @@ class TDTestCase: def __init__(self): self.snapshot = 0 self.vgroups = 2 - self.ctbNum = 1000 + self.ctbNum = 100 self.rowsPerTbl = 1000 def init(self, conn, logSql): @@ -38,9 +38,9 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 1000, + 'ctbNum': 100, 'rowsPerTbl': 1000, - 'batchNum': 100, + 'batchNum': 10, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 3, 'showMsg': 1, @@ -135,9 +135,9 @@ class TDTestCase: tdLog.info("================= restart dnode ===========================") tdDnodes.stop(1) tdDnodes.start(1) - time.sleep(3) + # time.sleep(3) - tdLog.info("insert process end, and start to check consume result") + tdLog.info(" restart taosd end and wait to check consume result") expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) totalConsumeRows = 0 @@ -186,7 +186,7 @@ class TDTestCase: # tdLog.info("****************************************************************************") - tmqCom.waitSubscriptionExit(tdSql) + tmqCom.waitSubscriptionExit(tdSql, topicFromStb1) tdSql.query("drop topic %s"%topicFromStb1) tdLog.printNoPrefix("======== test case 1 end ...... ") @@ -204,11 +204,11 @@ class TDTestCase: 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'DOUBLE', 'count':1},{'type': 'BINARY', 'len':32, 'count':1},{'type': 'NCHAR', 'len':32, 'count':1}], 'ctbPrefix': 'ctb', 'ctbStartIdx': 0, - 'ctbNum': 1000, + 'ctbNum': 100, 'rowsPerTbl': 1000, - 'batchNum': 3000, + 'batchNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - 'pollDelay': 5, + 'pollDelay': 20, 'showMsg': 1, 'showRow': 1, 'snapshot': 0} @@ -254,11 +254,11 @@ class TDTestCase: tdLog.info("================= restart dnode ===========================") tdDnodes.stop(1) tdDnodes.start(1) - time.sleep(3) + # time.sleep(3) - # tdLog.info("create some new child table and insert data ") - # paraDict["batchNum"] = 1000 - # paraDict["ctbPrefix"] = 'newCtb' + tdLog.info("create some new child table and insert data ") + paraDict["batchNum"] = 100 + paraDict["ctbPrefix"] = 'newCtb' # tmqCom.insert_data_with_autoCreateTbl(tdSql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) tdLog.info("insert process end, and start to check consume result") @@ -275,6 +275,7 @@ class TDTestCase: if totalConsumeRows != totalRowsFromQuery: tdLog.exit("tmq consume rows error!") + tmqCom.waitSubscriptionExit(tdSql, topicFromStb1) tdSql.query("drop topic %s"%topicFromStb1) tdLog.printNoPrefix("======== test case 2 end ...... ") From e00528eead368f72ad1b78c64409714a4689f067 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 09:59:37 +0800 Subject: [PATCH 25/36] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 5 ++--- source/libs/executor/src/executorimpl.c | 8 +------- source/libs/executor/src/scanoperator.c | 8 ++++++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 25063d1454..1ad17bbc76 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -869,7 +869,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re SExecTaskInfo* pTaskInfo); SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, - STimeWindowAggSupp* pTwAggSup, SExecTaskInfo* pTaskInfo); + SExecTaskInfo* pTaskInfo); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); @@ -879,8 +879,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 8162bb609d..2cf45f8ce5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4458,12 +4458,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return createExchangeOperatorInfo(pHandle->pMsgCb->clientRpc, (SExchangePhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - STimeWindowAggSupp aggSup = (STimeWindowAggSupp){ - .waterMark = pTableScanNode->watermark, - .calTrigger = pTableScanNode->triggerType, - .maxTs = INT64_MIN, - }; - if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, @@ -4483,7 +4477,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo #endif pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, &aggSup, pTaskInfo); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTaskInfo); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4a27b8a011..7d84a6ba93 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1416,7 +1416,7 @@ static void destroyStreamScanOperatorInfo(void* param, int32_t numOfOutput) { } SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, - STimeWindowAggSupp* pTwSup, SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo) { SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1430,7 +1430,11 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pTagCond = pTagCond; pInfo->pGroupTags = pTableScanNode->pGroupTags; - pInfo->twAggSup = *pTwSup; + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pTableScanNode->watermark, + .calTrigger = pTableScanNode->triggerType, + .maxTs = INT64_MIN, + }; int32_t numOfCols = 0; pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); From 0620c47687930c03a345ad6541f79e6c499bfa75 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 10:15:58 +0800 Subject: [PATCH 26/36] test: disable unit test. --- source/client/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index 0b259169dc..129e20e5de 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -51,6 +51,6 @@ target_link_libraries( PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom ) -#if(${BUILD_TEST}) +if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -#endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) \ No newline at end of file From 950be95fdfa66fd38be807191c6c502014ca4dc5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 10:40:58 +0800 Subject: [PATCH 27/36] fix(query): fix error in limit/offset --- source/libs/executor/src/executorimpl.c | 33 +++++++++++++++++++------ tests/script/tsim/parser/fill.sim | 1 + 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2cf45f8ce5..7bac828a53 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -3406,12 +3406,13 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } if (pProjectInfo->mergeDataBlocks) { - if (pFinalRes->info.rows + pInfo->pRes->info.rows <= pOperator->resultInfo.threshold) { - pFinalRes->info.groupId = pInfo->pRes->info.groupId; - pFinalRes->info.version = pInfo->pRes->info.version; + pFinalRes->info.groupId = pInfo->pRes->info.groupId; + pFinalRes->info.version = pInfo->pRes->info.version; - // continue merge data, ignore the group id - blockDataMerge(pFinalRes, pInfo->pRes); + // continue merge data, ignore the group id + blockDataMerge(pFinalRes, pInfo->pRes); + + if (pFinalRes->info.rows + pInfo->pRes->info.rows <= pOperator->resultInfo.threshold) { continue; } } @@ -4246,9 +4247,11 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, } SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { - int32_t numOfCols = LIST_LENGTH(pScanNode->pScanCols); + int32_t numOfCols = LIST_LENGTH(pScanNode->pScanCols); + int32_t numOfTags = LIST_LENGTH(pScanNode->pScanPseudoCols); + SSchemaWrapper* pqSw = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - pqSw->pSchema = taosMemoryCalloc(numOfCols, sizeof(SSchema)); + pqSw->pSchema = taosMemoryCalloc(numOfCols + numOfTags, sizeof(SSchema)); for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanCols, i); @@ -4261,6 +4264,22 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); } + // this the tags and pseudo function columns, we only keep the tag columns + for(int32_t i = 0; i < numOfTags; ++i) { + STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanPseudoCols, i); + + int32_t type = nodeType(pNode->pExpr); + if (type == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + + SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; + pSchema->colId = pColNode->colId; + pSchema->type = pColNode->node.resType.type; + pSchema->type = pColNode->node.resType.bytes; + strncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + } + } + return pqSw; } diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index 698314fa36..396bdd1e56 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -960,6 +960,7 @@ endi sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 4:2:15' interval(500a) fill(value, 99,91,90,89,88,87,86,85) ; if $rows != 21749 then + print expect 21749, actual: $rows return -1 endi From b4e3865a8ade860e7e8a4004661c30079c94affd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 23 Jul 2022 02:43:22 +0000 Subject: [PATCH 28/36] fix: delete stuck when replica is 3 --- source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f0746494e8..5463f42f6a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -112,8 +112,8 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { tEncodeSize(tEncodeDeleteRes, &res, size, ret); pCont = rpcMallocCont(size + sizeof(SMsgHead)); - ((SMsgHead *)pCont)->contLen = htonl(size + sizeof(SMsgHead)); - ((SMsgHead *)pCont)->vgId = htonl(TD_VID(pVnode)); + ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); + ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); tEncodeDeleteRes(pCoder, &res); From c055d1e3839f43e030426596b3acecaece1de0b8 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 23 Jul 2022 12:49:19 +0800 Subject: [PATCH 29/36] test:modify tmq consumer process --- tests/system-test/7-tmq/tmqDnodeRestart.py | 12 +- tests/system-test/fulltest.sh | 2 +- tests/test/c/tmqSim.c | 143 +++++++++++++-------- 3 files changed, 97 insertions(+), 60 deletions(-) diff --git a/tests/system-test/7-tmq/tmqDnodeRestart.py b/tests/system-test/7-tmq/tmqDnodeRestart.py index 9c4ba2be55..cec6985a4e 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart.py @@ -131,10 +131,10 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) # time.sleep(3) - tmqCom.getStartCommitNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("================= restart dnode ===========================") - tdDnodes.stop(1) - tdDnodes.start(1) + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) # time.sleep(3) tdLog.info(" restart taosd end and wait to check consume result") @@ -250,10 +250,10 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tmqCom.getStartCommitNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("================= restart dnode ===========================") - tdDnodes.stop(1) - tdDnodes.start(1) + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) # time.sleep(3) tdLog.info("create some new child table and insert data ") diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 543b1e2b34..738add34bf 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -221,7 +221,7 @@ python3 ./test.py -f 7-tmq/tmqDropStb.py python3 ./test.py -f 7-tmq/tmqDropStbCtb.py python3 ./test.py -f 7-tmq/tmqDropNtb.py python3 ./test.py -f 7-tmq/tmqUdf.py -# python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 190b4224a4..6a18263d50 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -352,6 +352,29 @@ void ltrim(char* str) { // return str; } +int queryDB(TAOS* taos, char* command) { + int retryCnt = 10; + int code; + TAOS_RES* pRes; + + while (retryCnt--) { + pRes = taos_query(taos, command); + code = taos_errno(pRes); + if (code != 0) { + taosSsleep(1); + taos_free_result(pRes); + continue; + } + taos_free_result(pRes); + return 0; + } + + pError("failed to reason:%s, sql: %s", tstrerror(code), command); + taos_free_result(pRes); + return -1; +} + + void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { int32_t i; for (i = 0; i < pInfo->numOfVgroups; i++) { @@ -374,30 +397,49 @@ void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { } } +TAOS* createNewTaosConnect() { + TAOS* taos = NULL; + int32_t retryCnt = 10; + + while (retryCnt--) { + TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (NULL != taos) { + return taos; + } + taosSsleep(1); + } + + taosFprintfFile(g_fp, "taos_connect() fail\n"); + return NULL; +} + + int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { char sqlStr[1100] = {0}; if (strlen(buf) > 1024) { taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); taosCloseFile(&g_fp); - exit(-1); + return -1; } TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not save consume result to main script\n"); + return -1; + } sprintf(sqlStr, "insert into %s.content_%d values (%" PRId64 ", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->ts++, buf); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - pError("error in insert consume result, reason:%s\n", taos_errstr(pRes)); - taosFprintfFile(g_fp, "error in insert consume result, reason:%s\n", taos_errstr(pRes)); + int retCode = queryDB(pConn, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "error in save consume content\n"); taosCloseFile(&g_fp); - taos_free_result(pRes); + taos_close(pConn); exit(-1); } - taos_free_result(pRes); + taos_close(pConn); return 0; } @@ -591,15 +633,12 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn int32_t code = tmq_get_raw_meta(msg, &raw); if(code == TSDB_CODE_SUCCESS){ - TAOS_RES* pRes = taos_query(pInfo->taos, "use metadb"); - if (taos_errno(pRes) != 0) { - pError("error when use metadb, reason:%s\n", taos_errstr(pRes)); - taosFprintfFile(g_fp, "error when use metadb, reason:%s\n", taos_errstr(pRes)); + int retCode = queryDB(pInfo->taos, "use metadb"); + if (retCode != 0) { + taosFprintfFile(g_fp, "error when use metadb\n"); taosCloseFile(&g_fp); - taos_free_result(pRes); exit(-1); } - taos_free_result(pRes); taosFprintfFile(g_fp, "raw:%p\n", &raw); taos_write_raw_meta(pInfo->taos, raw); @@ -618,19 +657,6 @@ static int32_t meta_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIn return totalRows; } - -int queryDB(TAOS* taos, char* command) { - TAOS_RES* pRes = taos_query(taos, command); - int code = taos_errno(pRes); - if (code != 0) { - pError("failed to reason:%s, sql: %s", tstrerror(code), command); - taos_free_result(pRes); - return -1; - } - taos_free_result(pRes); - return 0; -} - static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {} int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { @@ -720,15 +746,12 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { char tmpString[128]; taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId, sqlStr); - TAOS_RES* pRes = taos_query(pInfo->taos, sqlStr); - if (taos_errno(pRes) != 0) { - pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - exit(-1); + int retCode = queryDB(pInfo->taos, sqlStr); + if (retCode != 0) { + taosFprintfFile(g_fp, "consume id %d error in save consume result\n", pInfo->consumerId); + return -1; } - taos_free_result(pRes); - return 0; } @@ -823,18 +846,18 @@ void loop_consume(SThreadInfo* pInfo) { void* consumeThreadFunc(void* param) { SThreadInfo* pInfo = (SThreadInfo*)param; - pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + pInfo->taos = createNewTaosConnect(); if (pInfo->taos == NULL) { taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); - ASSERT(0); return NULL; } build_consumer(pInfo); build_topic_list(pInfo); if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { - taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); - assert(0); + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -842,7 +865,8 @@ void* consumeThreadFunc(void* param) { if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); - assert(0); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -926,17 +950,20 @@ void parseConsumeInfo() { int32_t getConsumeInfo() { char sqlStr[1024] = {0}; - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); + TAOS* pConn = createNewTaosConnect(); + if (pConn == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not get consume info for start consumer\n"); + return -1; + } sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); + TAOS_RES *pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { - pError("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); - taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error in get consumeinfo for %s\n", taos_errstr(pRes)); taosCloseFile(&g_fp); taos_free_result(pRes); - exit(-1); + taos_close(pConn); + return -1; } TAOS_ROW row = NULL; @@ -981,6 +1008,7 @@ int32_t getConsumeInfo() { taos_free_result(pRes); parseConsumeInfo(); + taos_close(pConn); return 0; } @@ -1123,7 +1151,6 @@ void* ombConsumeThreadFunc(void* param) { if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); - assert(0); return NULL; } @@ -1131,7 +1158,6 @@ void* ombConsumeThreadFunc(void* param) { if (err != 0) { pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); - assert(0); return NULL; } @@ -1181,9 +1207,9 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { void* ombProduceThreadFunc(void* param) { SThreadInfo* pInfo = (SThreadInfo*)param; - pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + pInfo->taos = createNewTaosConnect(); if (pInfo->taos == NULL) { - printf("taos_connect() fail\n"); + taosFprintfFile(g_fp, "taos_connect() fail, can not start producers!\n"); return NULL; } @@ -1200,6 +1226,8 @@ void* ombProduceThreadFunc(void* param) { char* sqlBuf = taosMemoryMalloc(MAX_SQL_LEN); if (NULL == sqlBuf) { printf("malloc fail for sqlBuf\n"); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -1232,6 +1260,8 @@ void* ombProduceThreadFunc(void* param) { int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); if (affectedRows < 0) { + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -1266,6 +1296,8 @@ void* ombProduceThreadFunc(void* param) { } printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); + taos_close(pInfo->taos); + pInfo->taos = NULL; return NULL; } @@ -1301,10 +1333,9 @@ void startOmbConsume() { taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); if (0 != g_stConfInfo.producers) { - TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + TAOS* taos = createNewTaosConnect(); if (taos == NULL) { - taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); - ASSERT(0); + taosFprintfFile(g_fp, "taos_connect() fail, can not create db, stbl, ctbl, topic!\n"); return ; } @@ -1357,9 +1388,11 @@ void startOmbConsume() { taosFprintfFile(g_fp, "==== close tmqlog ====\n"); taosCloseFile(&g_fp); + taos_close(taos); return; } + taos_close(taos); } // pthread_create one thread to consume @@ -1418,7 +1451,11 @@ int main(int32_t argc, char* argv[]) { return 0; } - getConsumeInfo(); + int32_t retCode = getConsumeInfo(); + if (0 != retCode) { + return -1; + } + saveConfigToLogFile(); tmqSetSignalHandle(); From 23ac4cbc93906e40917e425c57a42239b83d47e2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 23 Jul 2022 12:59:36 +0800 Subject: [PATCH 30/36] handle except --- source/libs/index/src/indexFilter.c | 2 +- source/libs/transport/src/.transSvr.c.swo | Bin 0 -> 53248 bytes source/libs/transport/src/transCli.c | 1 + source/libs/transport/src/transSvr.c | 57 +++++++++++----------- 4 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 source/libs/transport/src/.transSvr.c.swo diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 27c90af3e7..ba56350b38 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -708,7 +708,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } sifFreeRes(ctx.pRes); - + SIF_RET(code); } diff --git a/source/libs/transport/src/.transSvr.c.swo b/source/libs/transport/src/.transSvr.c.swo new file mode 100644 index 0000000000000000000000000000000000000000..c9486c50867994926ea395577110ee4d15de5632 GIT binary patch literal 53248 zcmeI53wT^tb?-%a7AT~=pY4fLyOJWyj|7L=0wbx#I@3q!md!Mh)P)+$?C_2p){vSqP4*V@u< zmTIj=z1iNJmtv`m@8|cE|nugzLw{-#a?)|8&Rw zrBI=m|9v~|f34&G{lfjl@b~cD{Y!lQ{^9y_!{3i|+z&td&I$LQ6c`qt50}^ftaD(U z1M3`E=fFA#);X}wfpre7b6}kV>l|3;z&Z#1OE^%P@9Wz@$vb2Zv;H6L|NnSkU*8A8 zGB_9f!vp&Iz6M?n_JLmnzX1N~XZreH0$vPW1Wtmdg9?}j=Ym1-K=1(Y9~jbK0q+5C z0j~tV2~L9Rz>~ox;9M{W9tysLasDOn2jJ!4Bv=3yFbM|10JuN+E{6W6!Kc9c!Mng~ zK?Pg_?gxI1v*1VI@4+X)AA`4mmw=nVb3qkc0nP_M2kr~Lj}zjH;5P6!@O$8O;8(zn zpazz}72qQ9i{K&P!Qek|aC{B?Irts$Yv5Nw3(SK3;6m_y93}q*z6L%IJ_k;K$AVAd zWO)O)4r~K=;A40*Xn{+?eZalJJ;0}NNPHaJ0-g%0;5=|2aBuJ#+WVD2ev(CS0gwzl z4y@tRYM0v8a&oL*ollmJ9jLdf3n%tfYu8=ctY25D&D2L4js9tZH%9BV+F(-V|AAc) zT6k)@U23+cE4BHlO3PF4Zygy-mRd(hTTD9HtTal^iW2WBwM&+DOgsbuvvbP}v+cp; z_*~bcg;K_A?Mky&TFf$Nd2t=(UDm8rM(6s;_-Ls%zgSV;{;ezjL#4&*dM6z+_6#Pa zrYh7?!XctyMi!uErqsGl1t}~>#WX9|4<*%FI}rz#TixZ6Zcn|(Q>07=w}7NwU8-z` z=dIL&pVftY zC+*b)mbcpVjO5*SGMS#PG>=u96LU|cTO@P!`XZuF)6CNo6Ujp=myMjr?n;s^Tau?P zBQkW8>zAwTWT9DK^8ZRE+OgA+34Q6j?T8v5MXjlz{IzPst{j_fAR@HVV4^RDYkHDv z`>HUiW|U29V3;}DoL6%GBE+E%8WZBDw!Ad4pp=#76e&(9)2{pebgT28dVZxjK7WI5 zP}|%{KpqSAz@No>y&=)k-#~M_%oRvz8EbWMX5I_oV%uol)z?1JsLWRu7`~dz<#w(e zk{g-1leB(%+?vJMtY4Z;RB1&4j?-&(?fT`)a>eEk<4!DZ|8l!>gJz2TZL5YSg9`g_ zUC&P2pfX9*4|3VsTR(J*_K)@BR<4Tm*6hN(H|YYRVktxOkucwY3@jRils0~4=++~V zNIkGJUt&84wlG&Rp&8l60DjEC$wchb)nW~;oava*%88XuCtY1_s7v)|6Duq$*;7%G zqot-ED_>VRad>8W&!w}IBh%A|CZ_hx?i)LBt@n;{wQ;o4)WkxnzU-k^rQEEvdp(v* zw2bU{l7k1vbD8RTGQKBSs5g6RFQ!oa+n#E(zICVeihc4N-;oWKwsO2P&5 zklHg)iq+hs z{BlW;(*vxEx~{|;Fz9@ZnX-)`GqY1;m(A`O8{IcDHP%1SJyy8Fmj7>=>(VOw}NDgh4$|`sCtdLumu`mn5L` z#_l~)E7LtSSnaRRFE3WcRqDmbEgvDt%HGVZ$iR$8pnxsF%cN9hXHR?=wJk2Fh5!^!x9NMeSqRFaF1 zw%d)JTeciOeth^yZF#ufJVFq4xl(IYT3dDxOAWlLzMQm<)|VIO(Or*KXzX&OdJG+~ z#E8{6k<=Hgt6hHJU^1c!y`EfNsa2Y##bk1Mj=Yk6o*#KByUWIB4vbAtCznl3B_qk? z$kfdE=s_AXnLIc(IWawEWsV%UDmgSVH8pZz=Biy*2b7hha;#FbvZ_ms#VQmWFEwFY z+i2K7HZ{7J^dpy!?;D@FiiT5ZGr5{4Fkzl4vRR#D+?Nh(nH{Zbd^}oeCoukab#XD7 zt0W903(Jc`$ZQYt8%8dl+_zz1h&dr?sH&}_Ls;f%_Ik~#M?w514~X7$L?cG6!SNVad?c40CzGBKUVRvJpiYvtkO zqNg5hm+F^*7WF9&m+MQrrTEMKe+72)ZtQH?|K|kb?=#r^?*K0V3!nn_fo)(bcp$g~ z`~L^v1K^$DRp6DN2JQ#Gg1!Gva0_@7cqMoW7zGak_XPh&xqk}e3wRZfFW?p6GB5^2 z*TrD{v(ABa4y@xaKp+6?{~xs^5yC6ybF^5m=+uzhD| zSyq#&%3`I|s*K&>6!*vKO;r~5A8GevJ(|bKDEP?{w{54~v4Im?t+w(woDOkDxVb{L z;LtCp^8DuAsXKHi*|fBI_w@x%Zfu!PQ>?`4(KKgx4^CpAE-ucM%3`UNvad8(S)_#M z!G%e%dh>2?87so2An)eIXA)@29yD(gWXVCpecV9SimE?XX@4%ZgCj`(w*9tnsEgRQ z`zuTJ=7~{gtC#z?`GQ1!Aujn1KnB^2OkZ_ibarfNYW9g!V>8YbCono@-TyvMrd9`| z(H>R7!}3TDn*@*`_#W}CS=FJ&sN>F{5t)`Zx_4~!Nwau6_Us$$C)>Ob(TB{DnW(+- zW%4u+Z{D4iWPY$Hkb@$tkjA;ZE9;DrZP3$Swso+SK3`_V(I)$!CGo!RX(++|kMaF~ zg?;~4@Juia{uO)vU%)58$HB|Mlfln}zr&8d9y|m*7H#7 z8@>Ua0UiY20s%XL*5c>D9juqX5d0jtl{N9pz?WG6u7RHduVC$a5c~=2+*9Co*0Y}m z9su6N`nA@v|AN5#rvTOKyQ*9IfxmoC`k$ZZ|JU@z?3XG_txCIp6Rl9&yqm6sTX-nh zn#yMWIszIWJB`0tS^a}qKia&zywDiP^+?7|+*&o9h?^u5DKW!{8ol+v!F~HqU3^wn z&))3$W<=bX>ZCQ#%8J9hve(Ge^Z|M(x@mn?Dk(J@3`a_I>WZYL_?K8#bpbzGo;h+T zYmPdDpIFw(GnnT}S;eyAL~@rJWTisF^;nj8c(zRnwjeOM+&UV?KP-PB{I(7me6w4D`7)Xm1xMt~U;-5#v4Ns9dobF?@?R!qOP|gr~ONaN*;&UvP3)63a?w zyf`DD(O&G;k7b<}Gwx*KSeA7yX~de`?Z+tk@MQ!m>}46Rzu#7{29EOm&dnzxuTa(~ zg5LCH!`P9->fvdRv%RVw%kpv87hP#qDn$&-8SZ1TEXjt+Rgfym6T?tD{rZ{c)v>Hz zbw1PAVp-iYWmxzNQzlaxX7;B+(Yk^Y)={o0}!2zYS2mQYO#Yf=nMu-H9bF|2lX zrj~VNY12TSU?-t-*`m3kWe${fvKZLf zKWHPj8Rwq-@U;WcP&E?`w@GG?QlWW-JGn8-ywy--oj-d8;cD5hFlr9Pum!O(O&X~> zMOj5^V>w;ny4^&s^q}?HJ5{JEcghE2UFDAI7?@TMMz-di&m;r>V4+!g5|lK8Idlr+ zW#=;;sR=<)&|z8AX0m0_{~k5(SBOm++(I7-Ip)ZNaDquCo-X-Ek@tD;P35DN@^#7a zqZPsrnn`I<@_j-zok!;-w4&0kBn|vAm3GY6C=C^xMvu%`7y_TwK?ptR`@A-)WB~zK3t&GvHm|CEzMB4JN=j;79lzJ_LRX zycs+Wd<_3X30w)jkN@EX;341(_#kcr9{}$MzYP>?zX{wMh~J+R&wnBxoT3_9(G6X> z?$VPRdri@ap=Es*H+9~yZ0TgFdV@lA6o1rE$Vy95RE)6Ohm*Quo?5sO6NU9Ckc44> zvFhH!Fe#hgwT%1{6sxG)r6qz_5a~q)3zE&Y;dHES5Diqe=$aU)r1CBQCVK)XE=(T& zWFWd$*9gQoxNS(DJh_T3LIRh;ME;1cp66jPOj>1LSjUEr+VQjLR%M$5@frvvro67hU!$OVy?g1i;C3mHcn zh+QL+2YS_}kko!Fq&B*>6-AwS^+*|t%&sY|cSrvps`pgpmXGvru<-z=U8zh^J<)86 zwu>5@IqWR2v=Ob#u25O1h$Y9$nwdD_S&IgOKE)9qiHKtIB$A zp{MymfX3K@T^t_LjEt#Fbs)y-qD~q-9w8(B60Pa&kaLfsa*0%cqD<3F*w92YYd)Va z%8|&j)&mTB@(e&kWD14JQqqIbHysJd`=#SGEt*6zAeWSAt6(qV#TXF=T4w_zze0j7 zTG%!__4oH@y6r&VT#SJp*t}a-#!#Y=Vkx!Kf_>Xl{FUyWGTM~AQM-cFgyPjQv#mC^ zj+B2a0pBnw^T?yjzo;7!Dtaq_>Gi^^lFB*k&aXt5av$eqVPA_Gg;XIS3Uq z6MH85=Q;iOMD=K?KCp95yS3cpku=d-<;IEF+Z8sSPG|muIxSXev7L3zi_YYihKJ&# z?Rt&1x>Ty8IHZzg?zuv$bb3skYI+*`Pk+d=zj%EBAu|EVZ;H{HmJ`sRhtSH}_u`uf zG``q}B;vT;pCdZG5786*xj>jQC zDh-o6gCLqjiT!2As_hf0k3>YIdn@Q;#zRx*T}FBrKN7N0v^@Ds=t2a2x}gwi5DPRH zbgiV-sFbS5}Kh`y#hZ+&fxYhwSaVl<40MjPx`kG@Jqi>{}cUCN9u4ZtmqR$u!c{3GJ4Aj zUW3%Q(jG8&03NB;i3Ue7H{jj!CpC-6B4_cOGziri%e-^mrOa2c5}($7}kZv3?f0sur5gZ;&33TbIMC z*_vehRE)XC*8eft`#v7*|7x&Dwf6s9a5cCJd|Bj9H6dhl#84}J;Ef^qOL@HsMj zJva=m1{=WH;CAf)Uj;7$F9er@3&A76_ptqM1HTVm1l7zY4q<41rDHQQ%?V zr@$T9{NDrL1+NCr2Q_dx=m-B3oBtN@WRQSIfCqseVC#Pud>gz9{0h+i`zL`Xg0sOR zz#Z89p9Y@W^0TicjKDZnB1N;NW z!FAwLa1Pi2ZpSb1QE)T(4e)GmHJAi@!KL6F@Ccyx`pZuLfttHL)@;_BnZ}*PLfJ0k zV6<;+g^5kt)NG~MoUP%r?RSOR6^4QZXR6gf_EhKW4^0*}(a;O~T+-s-?24|hB*_e? z;d!Mhl&1^DOxBu>>;6(9&c4{=8O+Y{4AGxVw@BS!16DCxq9$}LE^K)hYoQ54UtfF_ z-lkffR^AkcmZLxDpO)7o3T6oQ2zeC}W^7X4Q2~!r^C^Nua`Yq^IWtlMv|aYj#d;KS zbNx*Y{g^3gmXnoT^7%uAJFES^?|*dD$?v%~5u;t?q98N=w`H1qCi!~fuCiTp+g42A zoAPWEANg5GoR_Eh*s}Mf`7CNo=C79#?L{|^gTa`m*@-+I>B!2?MLUcwHQFaS1t2R2 zb8 zG;&~P${*(@Q(h}pNf~3!tw3#b)$+sDU{OVAxC-!8yLRc(R-dV`cXW#Iy?*1s zbk8*%mXNCT9V4K0X89{2+V%e;4R#Htx^F%z^JeJP8|{pOpu+b)c_EgK?!9^U9C4#+ zrF69Dibd2?*#H~&EIB5SHFW&JqubM(o7O!_!(4jT4_$yrjXXa3xSBgRYydpKPs#A!c_Qk{-!Au`Np%x-4gK`qxY z&L~4<#5RyyUtTspke+6K#gM)z`sUh)O3hlqLu>b$t?NxZvB>jkL>Fgv%UTmVqeLTsW zDV|W(;EttAy&dGKyUJ~`y7B;dSI(M=<0n&N>LQ^RHBrfem&XqrkWl+-cZX*MMou-# zL`$>RKqEDS>$a_YT!%wZT4PkKwLepBv|rUd3SKxsR`nb;48G}vqZsdr-XhH<|9~>EIrW@g%zdZ`h*%UmzO^_t_8FH=!qjF z*a=9V^e?meV?A*iZpNY!lK7QOHf76BS`xyo9R^5KYuF&)aB0!aY_y4dEe6iB+XG@H z(828(*Y;T4Q5HCzz!#hi?GIGwLbcdAgA zH1yH(2y@NarBv(xn7V!EVpD4U|JlL*{|9XTKL@t~odfuM&;+}H_W%Dow*EhZkAhpl zyMWFBXo5+w704g(O>F@;7ZodpA(&_C9Pck*Of8xq9kLvSIZ0ke##x;pw zoOr6eoTt*i>A3Gv3NGWb{nq&O?AZRvnXBw}7%`-G`+|ub6LqkLgZ#SwO&tEEy+iFA zY<%;_t@Lo`dg{fth)_&NCi{A1!FV@UYL|(UXx7V>R!fRb$qW|0sLUfRL%}e+uDW3 zB}S7DBBL{tiejTvEPDWA@%jfx-cc8R8}WDgf( zrGO|ewHm6PJqHOg0OIkOAsvRC>XTdZ45gu9Om+zxJ=$R%->gY(UxTu6>mcdG%J4-J1_bNQ+cJOTz3;0@GCN#I?3~VQ<;d0*ZPFk&dDmh# zWZ2D4KIY}TgL~=vNtlZUi!y(F&WtcJ3@g&QFz1dO2savo-o{T~!S==R=)Ot9M`Cqm zg2F_KF)L=BMKxultUc~)HJd>pTX{vftO1Qd4(ADnZ{e-9J;1BQgF&mse!2McaHLp+ zsT>)r(uDINNBh&Au1J+`s!wEAZDzmHKtU|BYt}f}&UP3lbVL~@(|sQKqQSDiVTDjL z#lsk1Jz_mw6!1MW#@c*)hq#>(3K231fr6nO()|dbTW2n4->2{dwkhnCvzl7@K;{24 zsiy%LYDg&W->QMzX}2A6%Drs5`V`Rk2YVK~o|+Lgy$VN!0SUTHSz zOUXW3YK=9fD;9EPEpH}oysA0|+Je|hJhOB;Sq{l^fw;28wc&dp_dqTl*M*7=mbqsLZ zHDTqZnSNAKsh; z{glz@a4j-rV$G*Y!xD`oKAQCE&#H!z9hc*xc4ErOE?fUkFz~ddEc-vcUr*=%eF(e{ zyaqf2=={Hnz!q>{@YCSS*!bETumCOscL#roU4JWh1JId&$HArGEO1|NFK`d=aqRoI zgO`FB8!*fFtAXAD5YPU*Ki_p`z@LCu0-aa*a&R>m1NQ`X10TcQe>YIv!0&_Sfd!y_ z1K+{cza4xHd=>mIcp11BTm^Q5v%!zB^*;vQ4Q>U$51tORpWtdBpTNVx!@%dT`TrYu zH+U9!BKSJC{D;7+!HdE3z)e6l{{Z*~cKp}DZ-cjh5ukX1Phzj%23`kV1%4Ae2@HWp zf*)b8e+>K?xEVCTG#CcjZ=iGi?*SA;@FDPe@GS65Fb7ItH&8o29GvRoBf#18DLekS zp@Uv(M`p`9Jhfk!f=fImKzr0x4Jz5C0nA&ZGiSt_fkdLvCGB=ccc#C)=4E0V+<)oC z!&|S_{OGH|{KzNr&=eQ)%ORoCRu?aFO~>W zy$Qi2ELkdXW-Wefj({D?rWj_Ijq~iXaSArOYe8NFBQd2bmeO%qU9IyHis3-AF=Swe z#JeMmv!;zuSVQfwy7-^x!%r<@#vS9{XE>X|!-(@;2zOP}XHp)89hwX?`_qG=MO5TJ zR|c{^)EyV?Z%PPjl5q}8W9%&R6?vdAVs&0<7&?;F!gJ&);gi_FR zmpbK^`r5YU7+yho9jS3;xsY22$diKPe)lD$BIk)0~3%~3HC zWL`PJ&gw{KmJXpY2{!^dN7SOuv)NX6(gDM_0C&EpB9?%jj5ylMP)?x%#(t>bN!DwO zVF?ae8k4q5^+BVP!PO4gURbV`?K~?I(O0d*ily+9BrJ<@+%iI9$?OST?ici#}dd>rRS=PVjz((*WEgRT9IIN8^i^eXqJ+dL*+^Ejy(6>6=nQnbaTs@UC3+c&C{l)uC+K#c&Qq4+ymQq@6SxX~O z{UHO?(+-po`e*o#-RfZ0-3`MtKu<1p5(yDHeB&-cvNpV-IJ2M@+7tFPIzw4Sm& zqmj&UZrkEfd2zam^qBEx(KPW+>YJG+la5H#J4Fkk6hZh)iSdiN3|q zEGUimYZ&6I+Lj%AOg9-((^;;ChOC!ukoI?CnJh_Z*ZTisFa{qYn=ohpe+AqAv*1(U zli-!$M({K+4W_^Vco6toY<|51;Qipu;1=*Ap!Wox1MUUx3BG{M|9PPO0eV;9m0%WdFY%`(A7RQ(zC+4YdCMEo}R5g0BGC{4WK+3SIP#{0Rx4~Dy{{lCI7lLPir-2>dtJwB`3-oTkYry$n1JHSdj|Jbs&Q~13d%?}% z_24<+N^lTd4$cLGK=A;I0T=>$hu@>Yqrf+@`M(T42>uk@3XX#v;1@sweja=s+h6Pa zPX*&(FX#t9#MXZUSOSN^As`>XXJ|*AHSluqQlLEq@&OzMJAvBy!C%{tnH=Hsumq^?xD4{ix#uj?kkAr&P$TwExzjokY($GA4x znRjtwZzdX@*fTaeH90yvIWaXeON79lsqq86NhPEWR&X|v^Og@nn7e0jGIbAu`e8r6 zD1ZsZlXvVA-DJ~gwBsJPZM(0gMfDYzNYvI=sYMrMbJZH5gPq#N3UoZ8GdO-gG~T?&ND)2B(aalYN^->`Mq{^b=08N%^d8|ypr)@zm01U7uS zu8~t4Qe%-jY8Hxg9m32OOYT-n*OTIFE~DVF?MkM>JYB@?3bIa%YkLFRSQ?H4=EW*T zik%iAH?Q{aMfHid?UeNv8IRZSIk<;7b~U7`70mvS)oC?rCs*<0=#TlMLaiYywu96`fi1k%X`bIajO&Jgla<~JhblVOE@mEkw#Vbc`yR&otGo}oN4U73dT*2;SAn`A}t zDyGxv!YI{fSN!DPs9x~)lx&r?tES7q8$B-vcpAGtar^Kpd zlqfePIhzPNFP=dbCYYaq%=YkU)BKrxB*<>=Xe^6}w^s%t&(;!o-l}E9RaKd{&&7Ep zb&+(+xOf@LkQJ2{S2X=qLg>OCQO0jEgH64;;kR>VFNwN`6rxY!8#M=4RRBun_w=!_N?f(+OITx#h_?A1tJ~5 z@^3u#@x}w)y??|W68Wrl)aO)HAVruz)O1D1&=J;~wLHP$NyJbyDVvFCBI805yPlK* z5^K7*?VFYk=k`n)gH0Z~4rL(Rx4cJ)^k$A%z&9Zip}!Jt?Z4@*`KWY_kIQsplXt3q=E35OTM*fThu8#=$c0hFYoehQ37f3V#ak%e zE107vn`ZQEm@@QSj~PQLT;QzMxm?0{AIi(rw61jkOOa}5DnmYNMI1S|Y!dWx;^{Et zU6=yXV(~3DhkKmZ0te-!)w@8MhYOQlaes-`Vhe$^MuvK(J*O8l#u%I|?kJKW2E+}s zHfeT8skyYM7GAnDJuGg9#Z}q<48=2}lSTzJM{!YPP?|^sxJ}@3O;*81z9icG%H-~yoc0DceK{=4AA z;LpHo!E3+~Z~$m8z!vaB?0mfw@GroxgKL1!3D^eo?tp*B&e!<>Hv*j*cmvQm0Ye}G zj{%PcKfvz49o!0@1D*)v6SxG(FYo~HkJ$RRf!_r>NAMPK99#nK3qFgT|61@oa0KWK z!GFQF{~Pev;8Q?*1LOnfT>szBwQb;EvF~pK9|Eri*MP&|=YZk>be3Q|JK!yRe-?Nq zm;-x&;sJgZoCW>{TmOCFW#A>?*T5vW5Ii2-fzAKN;00hOxDeb6d&YQuV`}9%_vaa1_fn_R#`nLre`NTL*_i}Y2-IA14nX}^l zB=cZLZEjI-74b!l!irns87}age`|Svk?Lv23)%;-FCLqk=|?}-GK#tb7lXFEq8YQP z+`eHLO%iK_axGL~wmhemBkZ2?ocG-ZRNGY^ft4MEg*mD? zLAent()O7HrgH-8B05&nhBzCpbDF9yvNrFvjN>5cZ;UQCD&VOcw7 zlNQd_p&FUjUVgW5WI~GUM z;6kixChx4^UT1$)ZMe`2$+a&x(JK1ZUHrKy_uMq6j7K_6EK6{yEbwM6?M+cFBf_z#Sc&5Z73GY*1Ck+PUugQ&XOvYF}&d zF<6|leMJpKd6KA}slB`z-1gLO38GSINV3&D^ABW-lHcptxm4^-K{`gIMCAv*jwQrk z@49Qu35&}_EJua^ajsd-BV&^+6%r7m3BiD;1c! zJBxaBn$U+vCMVyC#)eikPP!3aGm$On5ujA99V;zzD7cqOY5$2*6k1jti+Lvc38yOl znl3eQICDFAa9o^XYwCoYcv4IVY&P08UeO0l(9$Cs=v=k3thtAo)Si2 z_rzJf>FFFha${`j(4pKr8;3DX`&^+QlC7l|ufQ|gtt94oL*|b|PZhM#Rn2^}9y#OT zg$$RWD`@3e^_GJqwYaT2hYi~Et8LJxQw8Tq_|s$b>#=<)HT8Or8$E~R=sTqbejHOa zF8|jIF~LAiv<4r;3K&3AUZ*2Qe39Imh8AQZ7~5)g9F_TVhSh}OZW?8tuHVH|u0n!l z*E`S(H#CezF*Vgd7DIX#M3`z*4>~Db$Cx}gy*G8UOKx!ZgnV-fN5S1L5!DfnzR5Hl z1h4NdwL{U!<}ND%PGlmV3$p8B6dh}-&``zPWEy7&JMlmDerM(6oCt34_^Scv&}Qze z;OGsK=_+KGt5@s)nDBjH4(I|z+YpZzZbk2+yp9M9y}I&1Ka%TKx_GL1g`;?f?@DO zZ1it{_X64N$3O$<9RTNn`-Aslm%jtN6`TZha2nXHW&c|;1k&Ee+b?UPJ*X^dxEcEtG^4p3A`4Z0EfUu;L+f(X*adqmpab^q#YAJ zQ3H*GDE-b}rI~vnYWf<_M7vmu6OA5Ar%KnF`9F7rTOonZrAauvt#Hc&i93#Mvq{@J z+KT6x#d$T;B*W~Nd$!lR<)AY7w6hsM4p}{pe~WC&De9|ns;Iqps^kQbtL`>e=Xx)f z7$l5+He#d^^Ls7A3?;)_Cfnvq(%kwH3n(btnmfglj@_+k1Xz!XP;C$JH095B1@>>> zvVGu`mB>i7s-RS+>Lc2TJ85Iof2>1`RXIQGVWs!iIQ>DfDHv$p);($v<9I_T6i)XL zjTSC!hSF2kCT)_iAheQPgdjeJvAF~n#U{V!ytwGo(In&Xz*dMe{zY}F_>{=#+A$vr zc8hXIGLtUQbg0W|tyZc|&^}!4I~{2>vv6&VnW+xuF^CoR_6#jO63#y`A(p0)uFGkd zTBEx|l{|$5gCWl>Wb=2(#@F#`!5!m5s`$E7*QyUx$WJC#|C;9t;uzYx|RbOtFDrxNGf@N2SS?JgAnieC$wnw}PSSqdUL%;z-KexHvF9v47;rdZCEC-Fff1Q_+#Bu{akE zsoyatefP%S)prhFuj+y|C*!F;YC4?0)0M@AN%%rzkI#oOpjB%$k>>?lQdJHqOVTWd z5?-6Mv8CU`riTs_l0Cd}>*JriId^JYYo@Z)*i$9wU}n$kp7E*au}S2ff~QLhl__+- z{sHY8Q^=C7EvT%+lOr>GXW@Zl?Z&OVFYp|_M~P%>BkLMIPq^k9QFhHWIno+=!ipp$ z@L|-q?@anZRw*lrr0qEWTF9v`@XpHQ(D;Ep6Njdut5=h#2jv5dx)*_nMKuGTd0!pY z*X2zb^wY*RmkowfIH9pRn^YTIsW#Hzs;uOiy!4fx*`sTA$!glF*<&taUT)eEm2=t` zPA})|v0Tobi+tp7E=()dhh6rf7Ov*I7&r1iQznRQA~iC!Fgz7~#c2{zCHwyuu@$GW zCAI$l++hFzAME}A0d(e{_WbVvj|Gne4+sB>4e;yWTCfjX0Cs@C#b@w`K<5LT z0Clh(+zosme}T>lkT2m4;Ps#Zo&+|6yMzCW-{4!|GvJTFd%!EeN$_-VIncR)8^GD% zAMq#L3Z4(H1WyE)fV034Xn(!0@IBz|;3Uww0SAF(V+ySCqd7b1!hXIdgQM;IVjaSF zy2$+X!%Cvp`>2_~#W1+kT*b8WRh004m+d)i+e@&KE*NgBti_l=*9Mm}P13d;N!FcD zn<9I_qK&7StzwK)8orDplhX@5ZZtipNyeXe*2}`*<`#a=`)d`K^SWF(e97M0WS0gu zVY$g@;Z#@1(KQ>SDcBD*O0uJUbd;hmqVh+hP-gWr40UNnIlaQY17ZB{c$tm)DxFU0 zaty-r>^{~XGPSDM5BCjIaPc%*YhI&;O4<=M@ulyBMr1ZyxntjgY?a=r7~MNI`lQ*t zBM0{E8`F-qa1yMMgF>LQdJ;ZyQGhH1ud;HZ>_wk?mc|8qSD!RxYihOf6mL!>G_Vo^ z1k)pb%ykkyJ%z4x%Kn0J7@Q(u3FRlydG0z9|GC}8M80s2dfH(EuZqs2;4#nE*s)rQ z{Ro-KIz^nAp@MHsE>6G!5@TkeLbb9y{jCra_eAz^l6!O6f*<%z?{s z)KKlA*Uv%SGd9{?prv%eGAFl+_T2IT6XwRU(W4_czGr1)H@%a@jvqKv#7^bk8q}Lq zOp^U}GLJbB(S_kkpPOmL>nv)c>nnMOtxF#dq(g#rr`7c?P4q$YiVic23iRa=^T`ByWxwU94LkgU@}%~& zfiJf{!qAXOOV6d^{qp?>4|WeJ$ZSm4lcLs;=R3@a=EbiWxh!M=nH&bZMiO zbY^Pgz%=gzoSvDQxN6oXhRKPEeY3ouy6Xu)(V5C{{*g!dY3axE=@7s$`=%n;A$;L> zm-zJ>K`m|902<0qJJL!l%w#_|1Vy5wGn*!^EV1V=3!hor>$}$aK8mhbrr<#}&S~E4 X8I7VLU*SJlFp83@@A;kVNZtPjn%Mi$ literal 0 HcmV?d00001 diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 638067e7ef..2257458fc6 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -196,6 +196,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ transClearBuffer(&conn->readBuf); \ transFreeMsg(transContFromHead((char*)head)); \ + if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) return; \ tDebug("%s conn %p receive release request, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ if (T_REF_VAL_GET(conn) > 1) { \ transUnrefCliHandle(conn); \ diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 9b89847477..86f1c5df1e 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -149,34 +149,35 @@ static void* transAcceptThread(void* arg); static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static bool addHandleToAcceptloop(void* arg); -#define CONN_SHOULD_RELEASE(conn, head) \ - do { \ - if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - reallocConnRef(conn); \ - tTrace("conn %p received release request", conn); \ - \ - STraceId traceId = head->traceId; \ - conn->status = ConnRelease; \ - transClearBuffer(&conn->readBuf); \ - transFreeMsg(transContFromHead((char*)head)); \ - \ - STransMsg tmsg = {.code = 0, .info.handle = (void*)conn, .info.traceId = traceId, .info.ahandle = NULL}; \ - SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); \ - srvMsg->msg = tmsg; \ - srvMsg->type = Release; \ - srvMsg->pConn = conn; \ - if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ - return; \ - } \ - if (conn->regArg.init) { \ - tTrace("conn %p release, notify server app", conn); \ - STrans* pTransInst = conn->pTransInst; \ - (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); \ - memset(&conn->regArg, 0, sizeof(conn->regArg)); \ - } \ - uvStartSendRespInternal(srvMsg); \ - return; \ - } \ +#define CONN_SHOULD_RELEASE(conn, head) \ + do { \ + if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + reallocConnRef(conn); \ + tTrace("conn %p received release request", conn); \ + \ + STraceId traceId = head->traceId; \ + conn->status = ConnRelease; \ + transClearBuffer(&conn->readBuf); \ + transFreeMsg(transContFromHead((char*)head)); \ + \ + STransMsg tmsg = { \ + .code = 0, .info.handle = (void*)conn, .info.traceId = traceId, .info.ahandle = (void*)0x9527}; \ + SSvrMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSvrMsg)); \ + srvMsg->msg = tmsg; \ + srvMsg->type = Release; \ + srvMsg->pConn = conn; \ + if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ + return; \ + } \ + if (conn->regArg.init) { \ + tTrace("conn %p release, notify server app", conn); \ + STrans* pTransInst = conn->pTransInst; \ + (*pTransInst->cfp)(pTransInst->parent, &(conn->regArg.msg), NULL); \ + memset(&conn->regArg, 0, sizeof(conn->regArg)); \ + } \ + uvStartSendRespInternal(srvMsg); \ + return; \ + } \ } while (0) #define SRV_RELEASE_UV(loop) \ From b9b53f04c1c60c9e90787bf6c0058855368db4e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 13:18:36 +0800 Subject: [PATCH 31/36] fix(query):ignore null type. --- include/common/tdatablock.h | 4 ++-- source/common/src/tdatablock.c | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index a4f5904018..22aac46560 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -184,8 +184,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, - const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, + const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f7b0da0014..a5b34d89a0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -214,8 +214,8 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, - const SColumnInfoData* pSource, uint32_t numOfRow2) { +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, + const SColumnInfoData* pSource, int32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { return numOfRow1; @@ -369,6 +369,10 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); + if (pCol1->info.type == 0) { // ignore null type + continue; + } + capacity = pDest->info.capacity; colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); } From 112642a3a45587651379335efb2a1b596b33ac21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 13:20:39 +0800 Subject: [PATCH 32/36] test: temporarily disable two cases. --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1ee254af49..98bc138f7e 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -99,7 +99,7 @@ ./test.sh -f tsim/parser/commit.sim # TD-17661 ./test.sh -f tsim/parser/condition.sim ./test.sh -f tsim/parser/constCol.sim -./test.sh -f tsim/parser/create_db.sim +#./test.sh -f tsim/parser/create_db.sim ./test.sh -f tsim/parser/create_mt.sim # TD-17653 ./test.sh -f tsim/parser/create_tb_with_tag_name.sim ./test.sh -f tsim/parser/create_tb.sim @@ -223,7 +223,7 @@ # ---- stream ./test.sh -f tsim/stream/basic0.sim -./test.sh -f tsim/stream/basic1.sim +#./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim ./test.sh -f tsim/stream/distributeInterval0.sim From 19acda27b29232b6ab353c7ee3c8ba4bd7f76602 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 23 Jul 2022 13:32:25 +0800 Subject: [PATCH 33/36] test: add comment for tq --- source/dnode/vnode/src/tq/tq.c | 2 ++ source/dnode/vnode/src/tq/tqMeta.c | 1 + 2 files changed, 3 insertions(+) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 9f80bc50a4..002b629660 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -569,8 +569,10 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { taosArrayDestroy(tbUidList); } taosHashPut(pTq->handles, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); + tqDebug("try to persist handle %s consumer %ld", req.subKey, pHandle->consumerId); if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { // TODO + ASSERT(0); } } else { /*ASSERT(pExec->consumerId == req.oldConsumerId);*/ diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 86b26339bc..d282036fe3 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -101,6 +101,7 @@ int32_t tqMetaOpen(STQ* pTq) { handle.execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } + tqDebug("tq restore %s consumer %ld", handle.subKey, handle.consumerId); taosHashPut(pTq->handles, pKey, kLen, &handle, sizeof(STqHandle)); } From 3876bf72120b6a41d83e4da87e1301eb89c90a7b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 23 Jul 2022 13:54:28 +0800 Subject: [PATCH 34/36] handle except --- source/libs/transport/src/transCli.c | 5 ++++- source/libs/transport/src/transSvr.c | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 2257458fc6..da59dc605f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -196,7 +196,10 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ transClearBuffer(&conn->readBuf); \ transFreeMsg(transContFromHead((char*)head)); \ - if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) return; \ + if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) { \ + SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, 0); \ + if (cliMsg->type == Release) return; \ + } \ tDebug("%s conn %p receive release request, ref:%d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ if (T_REF_VAL_GET(conn) > 1) { \ transUnrefCliHandle(conn); \ diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 86f1c5df1e..ac14e22a51 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -397,11 +397,11 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { if (pConn->status == ConnNormal) { pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); + if (smsg->type == Release) pHead->msgType = 0; } else { if (smsg->type == Release) { pHead->msgType = 0; pConn->status = ConnNormal; - destroyConnRegArg(pConn); transUnrefSrvHandle(pConn); } else { From 4247cac9705474a9fbfb6bf44e2f32acd95cf431 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 23 Jul 2022 14:07:20 +0800 Subject: [PATCH 35/36] shell: fix shell history error --- tools/shell/src/shellCommand.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index ce3718e001..1af6ee4c06 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -524,10 +524,8 @@ int32_t shellReadCommand(char *command) { c = taosGetConsoleChar(); switch (c) { case 'A': // Up arrow - if (hist_counter != pHistory->hstart) { - hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; - shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]); - } + hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; + shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]); break; case 'B': // Down arrow if (hist_counter != pHistory->hend) { From 72574b69de9104615c7b1d6f47c35e930a2cadcb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Jul 2022 14:09:26 +0800 Subject: [PATCH 36/36] test: update the block merge. --- source/common/src/tdatablock.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a5b34d89a0..f64a0df36f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -263,7 +263,8 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->varmeta.length = len + oldLen; } else { if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { - ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); + // all data may be null, when the pColumnInfoData->info.type == 0, bytes == 0; +// ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -369,10 +370,6 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); - if (pCol1->info.type == 0) { // ignore null type - continue; - } - capacity = pDest->info.capacity; colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); }