From 006a3c38045ce840fd8eec6d396d46d0cab710d4 Mon Sep 17 00:00:00 2001 From: menshibin Date: Fri, 12 Jan 2024 14:14:03 +0800 Subject: [PATCH 1/6] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 104 ++++++++++++++++++++ tests/army/frame/autogen.py | 61 ++++++------ tests/army/frame/caseBase.py | 14 +-- tests/army/frame/server/cluster.py | 11 ++- tests/army/frame/server/dnodes.py | 5 + tests/army/test.py | 21 ++-- 6 files changed, 165 insertions(+), 51 deletions(-) create mode 100644 tests/army/community/cluster/incSnapshot.py diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py new file mode 100644 index 0000000000..a91e51cda4 --- /dev/null +++ b/tests/army/community/cluster/incSnapshot.py @@ -0,0 +1,104 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame.server.dnodes import * +from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=3): + super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1") + self.valgrind = 0 + self.childtable_count = 10 + # tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.prepare() + autoGen = AutoGen() + autoGen.create_db(self.db, 2, 3) + autoGen.create_stable(self.stb, 5, 10, 8, 8) + autoGen.create_child(self.stb, "d", self.childtable_count) + autoGen.insert_data(1) + tdSql.execute(f"flush database {self.db}") + clusterDnodes.stoptaosd(3) + clusterDnodes.stoptaosd(1) + clusterDnodes.starttaosd(3) + time.sleep(5) + clusterDnodes.stoptaosd(2) + clusterDnodes.starttaosd(1) + time.sleep(5) + autoGen.insert_data(10, 1600000000001) + # tdSql.execute(f"flush database {self.db}") + + sql = 'show vnodes;' + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + self.snapshotAgg() + + clusterDnodes.stopAll() + # for i in range(1, 4): + # path = clusterDnodes.getDnodeDir(i) + # dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") + # dirs = glob.glob(dnodesRootDir) + # for dir in dirs: + # if os.path.isdir(dir): + # tdLog.debug("delete dir: %s " % (dnodesRootDir)) + # self.remove_directory(os.path.join(dir, "wal")) + + clusterDnodes.starttaosd(1) + clusterDnodes.starttaosd(2) + clusterDnodes.starttaosd(3) + + time.sleep(3) + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'offline': + tdLog.exit( + "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1])) + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + + self.timestamp_step = 1 + self.insert_rows = 11 + self.checkInsertCorrect() + self.checkAggCorrect() + + def remove_directory(self, directory): + try: + shutil.rmtree(directory) + tdLog.debug("delete dir: %s " % (directory)) + except OSError as e: + tdLog.exit("delete fail dir: %s " % (directory)) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index 9dca96e7b0..a641deaf00 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -30,7 +30,7 @@ class AutoGen: # _columns_sql def gen_columns_sql(self, pre, cnt, binary_len, nchar_len): - types = [ + types = [ 'timestamp', 'tinyint', 'smallint', @@ -58,33 +58,33 @@ class AutoGen: sqls += "," sqls += sql metas.append(sel) - - return metas, sqls; + + return metas, sqls; # gen tags data def gen_data(self, i, marr): - datas = "" + datas = "" for c in marr: data = "" - if c == 0 : # timestamp + if c == 0: # timestamp data = "%d" % (self.ts + i) - elif c <= 4 : # small - data = "%d"%(i%128) - elif c <= 8 : # int + elif c <= 4: # small + data = "%d" % (i % 128) + elif c <= 8: # int data = f"{i}" - elif c <= 10 : # float - data = "%f"%(i+i/1000) - elif c <= 11 : # bool - data = "%d"%(i%2) - elif c == 12 : # binary + elif c <= 10: # float + data = "%f" % (i + i / 1000) + elif c <= 11: # bool + data = "%d" % (i % 2) + elif c == 12: # binary data = '"' + self.random_string(self.bin_len) + '"' - elif c == 13 : # binary + elif c == 13: # binary data = '"' + self.random_string(self.nch_len) + '"' if datas != "": datas += "," datas += data - + return datas # generate specail wide random string @@ -93,11 +93,11 @@ class AutoGen: return ''.join(random.choice(letters) for i in range(count)) # create db - def create_db(self, dbname, vgroups = 2, replica = 1): - self.dbname = dbname + def create_db(self, dbname, vgroups=2, replica=1): + self.dbname = dbname tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}') tdSql.execute(f'use {dbname}') - + # create table or stable def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len): self.bin_len = binary_len @@ -109,7 +109,7 @@ class AutoGen: sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})" tdSql.execute(sql) - # create child table + # create child table def create_child(self, stbname, prename, cnt): self.child_cnt = cnt self.child_name = prename @@ -120,7 +120,7 @@ class AutoGen: tdLog.info(f"create child tables {cnt} ok") - def insert_data_child(self, child_name, cnt, batch_size, step): + def insert_data_child(self, child_name, cnt, batch_size, step): values = "" print("insert child data") ts = self.ts @@ -130,7 +130,7 @@ class AutoGen: value = self.gen_data(i, self.mcols) ts += step values += f"({ts},{value}) " - if batch_size == 1 or (i > 0 and i % batch_size == 0) : + if batch_size == 1 or (i > 0 and i % batch_size == 0): sql = f"insert into {child_name} values {values}" tdSql.execute(sql) values = "" @@ -138,18 +138,25 @@ class AutoGen: # end batch if values != "": sql = f"insert into {child_name} values {values}" + tdLog.info(f" insert child data SQL{sql}") tdSql.execute(sql) tdLog.info(f" insert data i={i}") values = "" - tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + tdLog.info(f" insert child data {child_name} finished, insert rows={cnt} last_ts={ts}") + return ts - # insert data - def insert_data(self, cnt): + # insert data + def insert_data(self, cnt, bContinue=False): + if not bContinue: + self.ts = 1600000000000 + + currTs = 1600000000000 for i in range(self.child_cnt): name = f"{self.child_name}{i}" - self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") # insert same timestamp to all childs @@ -158,6 +165,4 @@ class AutoGen: name = f"{self.child_name}{i}" self.insert_data_child(name, cnt, self.batch_size, 0) - tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}") - - + tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 8d2c1e6d18..863a117cdd 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -28,7 +28,7 @@ class TBase: # # init - def init(self, conn, logSql, replicaVar=1): + def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"): # save param self.replicaVar = int(replicaVar) tdSql.init(conn.cursor(), True) @@ -40,14 +40,14 @@ class TBase: self.mLevelDisk = 0 # test case information - self.db = "db" - self.stb = "stb" + self.db = db + self.stb = stb # sql - self.sqlSum = f"select sum(ic) from {self.stb}" - self.sqlMax = f"select max(ic) from {self.stb}" - self.sqlMin = f"select min(ic) from {self.stb}" - self.sqlAvg = f"select avg(ic) from {self.stb}" + self.sqlSum = f"select sum({checkColName}) from {self.stb}" + self.sqlMax = f"select max({checkColName}) from {self.stb}" + self.sqlMin = f"select min({checkColName}) from {self.stb}" + self.sqlAvg = f"select avg({checkColName}) from {self.stb}" self.sqlFirst = f"select first(ts) from {self.stb}" self.sqlLast = f"select last(ts) from {self.stb}" diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py index ade8ac39a2..9be7fe56b1 100644 --- a/tests/army/frame/server/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -13,23 +13,24 @@ from frame.common import * class ClusterDnodes(TDDnodes): """rewrite TDDnodes and make MyDdnodes as TDDnodes child class""" - def __init__(self ,dnodes_lists): - + def __init__(self): super(ClusterDnodes,self).__init__() - self.dnodes = dnodes_lists # dnode must be TDDnode instance self.simDeployed = False self.testCluster = False self.valgrind = 0 self.killValgrind = 1 + def init(self, dnodes_lists, deployPath, masterIp): + self.dnodes = dnodes_lists # dnode must be TDDnode instance + super(ClusterDnodes, self).init(deployPath, masterIp) +clusterDnodes = ClusterDnodes() class ConfigureyCluster: """This will create defined number of dnodes and create a cluster. at the same time, it will return TDDnodes list: dnodes, """ hostname = socket.gethostname() - def __init__(self): - self.dnodes = [] + self.dnodes = [] self.dnodeNums = 5 self.independent = True self.startPort = 6030 diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 0d40b665dd..b58deeb789 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -899,6 +899,11 @@ class TDDnodes: dnodesRootDir = "%s/sim" % (self.path) return dnodesRootDir + def getDnodeDir(self, index): + self.check(index) + dnodesDir = "%s/sim/dnode%d" % (self.path, index) + return dnodesDir + def getSimCfgPath(self): return self.sim.getCfgDir() diff --git a/tests/army/test.py b/tests/army/test.py index a3e28b772d..37fecea370 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -405,11 +405,11 @@ if __name__ == "__main__": else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() for dnode in tdDnodes.dnodes: tdDnodes.deploy(dnode.index, updateCfgDict) for dnode in tdDnodes.dnodes: @@ -591,12 +591,11 @@ if __name__ == "__main__": print(independentMnode,"independentMnode valuse") # create dnode list dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.setAsan(asan) - tdDnodes.stopAll() + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() for dnode in tdDnodes.dnodes: tdDnodes.deploy(dnode.index,updateCfgDict) for dnode in tdDnodes.dnodes: From 2be6098a2e3e7fdd539ccb6e9e029f14f0fd00cb Mon Sep 17 00:00:00 2001 From: menshibin Date: Mon, 15 Jan 2024 14:59:09 +0800 Subject: [PATCH 2/6] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 58 ++++++++++----------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index a91e51cda4..94013eea70 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -31,45 +31,45 @@ class TDTestCase(TBase): autoGen.create_db(self.db, 2, 3) autoGen.create_stable(self.stb, 5, 10, 8, 8) autoGen.create_child(self.stb, "d", self.childtable_count) - autoGen.insert_data(1) + autoGen.insert_data(1000) tdSql.execute(f"flush database {self.db}") clusterDnodes.stoptaosd(3) - clusterDnodes.stoptaosd(1) - clusterDnodes.starttaosd(3) - time.sleep(5) - clusterDnodes.stoptaosd(2) - clusterDnodes.starttaosd(1) - time.sleep(5) - autoGen.insert_data(10, 1600000000001) - # tdSql.execute(f"flush database {self.db}") + # clusterDnodes.stoptaosd(1) + # clusterDnodes.starttaosd(3) + # time.sleep(5) + # clusterDnodes.stoptaosd(2) + # clusterDnodes.starttaosd(1) + # time.sleep(5) + autoGen.insert_data(5000, 1600000000001) + tdSql.execute(f"flush database {self.db}") - sql = 'show vnodes;' - while True: - bFinish = True - param_list = tdSql.query(sql, row_tag=True) - for param in param_list: - if param[3] == 'leading' or param[3] == 'following': - bFinish = False - break - if bFinish: - break + # sql = 'show vnodes;' + # while True: + # bFinish = True + # param_list = tdSql.query(sql, row_tag=True) + # for param in param_list: + # if param[3] == 'leading' or param[3] == 'following': + # bFinish = False + # break + # if bFinish: + # break self.snapshotAgg() - + time.sleep(10) clusterDnodes.stopAll() - # for i in range(1, 4): - # path = clusterDnodes.getDnodeDir(i) - # dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") - # dirs = glob.glob(dnodesRootDir) - # for dir in dirs: - # if os.path.isdir(dir): - # tdLog.debug("delete dir: %s " % (dnodesRootDir)) - # self.remove_directory(os.path.join(dir, "wal")) + for i in range(1, 4): + path = clusterDnodes.getDnodeDir(i) + dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") + dirs = glob.glob(dnodesRootDir) + for dir in dirs: + if os.path.isdir(dir): + tdLog.debug("delete dir: %s " % (dnodesRootDir)) + self.remove_directory(os.path.join(dir, "wal")) clusterDnodes.starttaosd(1) clusterDnodes.starttaosd(2) clusterDnodes.starttaosd(3) - time.sleep(3) + time.sleep(10) while True: bFinish = True param_list = tdSql.query(sql, row_tag=True) From 1119d95a4aa9166f9f06bc4a4a3571c5c7cf4c71 Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 14:38:53 +0800 Subject: [PATCH 3/6] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 2 +- tests/parallel_test/cases.task | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 94013eea70..21f67b45fc 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -40,7 +40,7 @@ class TDTestCase(TBase): # clusterDnodes.stoptaosd(2) # clusterDnodes.starttaosd(1) # time.sleep(5) - autoGen.insert_data(5000, 1600000000001) + autoGen.insert_data(5000, True) tdSql.execute(f"flush database {self.db}") # sql = 'show vnodes;' diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 63745f75ab..5e8921768d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -11,6 +11,7 @@ ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py From 7db577d664d4ec7d69678210a1f3f178eab511de Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 15:28:19 +0800 Subject: [PATCH 4/6] add inc Snapshot copy case --- tests/army/test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/army/test.py b/tests/army/test.py index 37fecea370..62b3b46a61 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -410,10 +410,10 @@ if __name__ == "__main__": clusterDnodes.setValgrind(valgrind) clusterDnodes.setAsan(asan) clusterDnodes.stopAll() - for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index, updateCfgDict) - for dnode in tdDnodes.dnodes: - tdDnodes.starttaosd(dnode.index) + for dnode in clusterDnodes.dnodes: + clusterDnodes.deploy(dnode.index, updateCfgDict) + for dnode in clusterDnodes.dnodes: + clusterDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) if restful or websocket: @@ -596,10 +596,10 @@ if __name__ == "__main__": clusterDnodes.setValgrind(valgrind) clusterDnodes.setAsan(asan) clusterDnodes.stopAll() - for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index,updateCfgDict) - for dnode in tdDnodes.dnodes: - tdDnodes.starttaosd(dnode.index) + for dnode in clusterDnodes.dnodes: + clusterDnodes.deploy(dnode.index,updateCfgDict) + for dnode in clusterDnodes.dnodes: + clusterDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) if restful or websocket: From 24066a7734cabcf4070e61633cba92bed78654cc Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 15:35:33 +0800 Subject: [PATCH 5/6] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 21f67b45fc..2095b6008b 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -68,7 +68,7 @@ class TDTestCase(TBase): clusterDnodes.starttaosd(1) clusterDnodes.starttaosd(2) clusterDnodes.starttaosd(3) - + sql = "show vnodes;" time.sleep(10) while True: bFinish = True From 022965661a7fcd0fa0a04823105b72aef7daeaad Mon Sep 17 00:00:00 2001 From: menshibin Date: Tue, 16 Jan 2024 15:38:34 +0800 Subject: [PATCH 6/6] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py index 2095b6008b..9f50ea52f1 100644 --- a/tests/army/community/cluster/incSnapshot.py +++ b/tests/army/community/cluster/incSnapshot.py @@ -84,7 +84,7 @@ class TDTestCase(TBase): break self.timestamp_step = 1 - self.insert_rows = 11 + self.insert_rows = 6000 self.checkInsertCorrect() self.checkAggCorrect()