From 006a3c38045ce840fd8eec6d396d46d0cab710d4 Mon Sep 17 00:00:00 2001 From: menshibin Date: Fri, 12 Jan 2024 14:14:03 +0800 Subject: [PATCH] add inc Snapshot copy case --- tests/army/community/cluster/incSnapshot.py | 104 ++++++++++++++++++++ tests/army/frame/autogen.py | 61 ++++++------ tests/army/frame/caseBase.py | 14 +-- tests/army/frame/server/cluster.py | 11 ++- tests/army/frame/server/dnodes.py | 5 + tests/army/test.py | 21 ++-- 6 files changed, 165 insertions(+), 51 deletions(-) create mode 100644 tests/army/community/cluster/incSnapshot.py diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py new file mode 100644 index 0000000000..a91e51cda4 --- /dev/null +++ b/tests/army/community/cluster/incSnapshot.py @@ -0,0 +1,104 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame.server.dnodes import * +from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=3): + super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1") + self.valgrind = 0 + self.childtable_count = 10 + # tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.prepare() + autoGen = AutoGen() + autoGen.create_db(self.db, 2, 3) + autoGen.create_stable(self.stb, 5, 10, 8, 8) + autoGen.create_child(self.stb, "d", self.childtable_count) + autoGen.insert_data(1) + tdSql.execute(f"flush database {self.db}") + clusterDnodes.stoptaosd(3) + clusterDnodes.stoptaosd(1) + clusterDnodes.starttaosd(3) + time.sleep(5) + clusterDnodes.stoptaosd(2) + clusterDnodes.starttaosd(1) + time.sleep(5) + autoGen.insert_data(10, 1600000000001) + # tdSql.execute(f"flush database {self.db}") + + sql = 'show vnodes;' + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + self.snapshotAgg() + + clusterDnodes.stopAll() + # for i in range(1, 4): + # path = clusterDnodes.getDnodeDir(i) + # dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") + # dirs = glob.glob(dnodesRootDir) + # for dir in dirs: + # if os.path.isdir(dir): + # tdLog.debug("delete dir: %s " % (dnodesRootDir)) + # self.remove_directory(os.path.join(dir, "wal")) + + clusterDnodes.starttaosd(1) + clusterDnodes.starttaosd(2) + clusterDnodes.starttaosd(3) + + time.sleep(3) + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'offline': + tdLog.exit( + "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1])) + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + + self.timestamp_step = 1 + self.insert_rows = 11 + self.checkInsertCorrect() + self.checkAggCorrect() + + def remove_directory(self, directory): + try: + shutil.rmtree(directory) + tdLog.debug("delete dir: %s " % (directory)) + except OSError as e: + tdLog.exit("delete fail dir: %s " % (directory)) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index 9dca96e7b0..a641deaf00 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -30,7 +30,7 @@ class AutoGen: # _columns_sql def gen_columns_sql(self, pre, cnt, binary_len, nchar_len): - types = [ + types = [ 'timestamp', 'tinyint', 'smallint', @@ -58,33 +58,33 @@ class AutoGen: sqls += "," sqls += sql metas.append(sel) - - return metas, sqls; + + return metas, sqls; # gen tags data def gen_data(self, i, marr): - datas = "" + datas = "" for c in marr: data = "" - if c == 0 : # timestamp + if c == 0: # timestamp data = "%d" % (self.ts + i) - elif c <= 4 : # small - data = "%d"%(i%128) - elif c <= 8 : # int + elif c <= 4: # small + data = "%d" % (i % 128) + elif c <= 8: # int data = f"{i}" - elif c <= 10 : # float - data = "%f"%(i+i/1000) - elif c <= 11 : # bool - data = "%d"%(i%2) - elif c == 12 : # binary + elif c <= 10: # float + data = "%f" % (i + i / 1000) + elif c <= 11: # bool + data = "%d" % (i % 2) + elif c == 12: # binary data = '"' + self.random_string(self.bin_len) + '"' - elif c == 13 : # binary + elif c == 13: # binary data = '"' + self.random_string(self.nch_len) + '"' if datas != "": datas += "," datas += data - + return datas # generate specail wide random string @@ -93,11 +93,11 @@ class AutoGen: return ''.join(random.choice(letters) for i in range(count)) # create db - def create_db(self, dbname, vgroups = 2, replica = 1): - self.dbname = dbname + def create_db(self, dbname, vgroups=2, replica=1): + self.dbname = dbname tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}') tdSql.execute(f'use {dbname}') - + # create table or stable def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len): self.bin_len = binary_len @@ -109,7 +109,7 @@ class AutoGen: sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})" tdSql.execute(sql) - # create child table + # create child table def create_child(self, stbname, prename, cnt): self.child_cnt = cnt self.child_name = prename @@ -120,7 +120,7 @@ class AutoGen: tdLog.info(f"create child tables {cnt} ok") - def insert_data_child(self, child_name, cnt, batch_size, step): + def insert_data_child(self, child_name, cnt, batch_size, step): values = "" print("insert child data") ts = self.ts @@ -130,7 +130,7 @@ class AutoGen: value = self.gen_data(i, self.mcols) ts += step values += f"({ts},{value}) " - if batch_size == 1 or (i > 0 and i % batch_size == 0) : + if batch_size == 1 or (i > 0 and i % batch_size == 0): sql = f"insert into {child_name} values {values}" tdSql.execute(sql) values = "" @@ -138,18 +138,25 @@ class AutoGen: # end batch if values != "": sql = f"insert into {child_name} values {values}" + tdLog.info(f" insert child data SQL{sql}") tdSql.execute(sql) tdLog.info(f" insert data i={i}") values = "" - tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + tdLog.info(f" insert child data {child_name} finished, insert rows={cnt} last_ts={ts}") + return ts - # insert data - def insert_data(self, cnt): + # insert data + def insert_data(self, cnt, bContinue=False): + if not bContinue: + self.ts = 1600000000000 + + currTs = 1600000000000 for i in range(self.child_cnt): name = f"{self.child_name}{i}" - self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") # insert same timestamp to all childs @@ -158,6 +165,4 @@ class AutoGen: name = f"{self.child_name}{i}" self.insert_data_child(name, cnt, self.batch_size, 0) - tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}") - - + tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 8d2c1e6d18..863a117cdd 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -28,7 +28,7 @@ class TBase: # # init - def init(self, conn, logSql, replicaVar=1): + def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"): # save param self.replicaVar = int(replicaVar) tdSql.init(conn.cursor(), True) @@ -40,14 +40,14 @@ class TBase: self.mLevelDisk = 0 # test case information - self.db = "db" - self.stb = "stb" + self.db = db + self.stb = stb # sql - self.sqlSum = f"select sum(ic) from {self.stb}" - self.sqlMax = f"select max(ic) from {self.stb}" - self.sqlMin = f"select min(ic) from {self.stb}" - self.sqlAvg = f"select avg(ic) from {self.stb}" + self.sqlSum = f"select sum({checkColName}) from {self.stb}" + self.sqlMax = f"select max({checkColName}) from {self.stb}" + self.sqlMin = f"select min({checkColName}) from {self.stb}" + self.sqlAvg = f"select avg({checkColName}) from {self.stb}" self.sqlFirst = f"select first(ts) from {self.stb}" self.sqlLast = f"select last(ts) from {self.stb}" diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py index ade8ac39a2..9be7fe56b1 100644 --- a/tests/army/frame/server/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -13,23 +13,24 @@ from frame.common import * class ClusterDnodes(TDDnodes): """rewrite TDDnodes and make MyDdnodes as TDDnodes child class""" - def __init__(self ,dnodes_lists): - + def __init__(self): super(ClusterDnodes,self).__init__() - self.dnodes = dnodes_lists # dnode must be TDDnode instance self.simDeployed = False self.testCluster = False self.valgrind = 0 self.killValgrind = 1 + def init(self, dnodes_lists, deployPath, masterIp): + self.dnodes = dnodes_lists # dnode must be TDDnode instance + super(ClusterDnodes, self).init(deployPath, masterIp) +clusterDnodes = ClusterDnodes() class ConfigureyCluster: """This will create defined number of dnodes and create a cluster. at the same time, it will return TDDnodes list: dnodes, """ hostname = socket.gethostname() - def __init__(self): - self.dnodes = [] + self.dnodes = [] self.dnodeNums = 5 self.independent = True self.startPort = 6030 diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 0d40b665dd..b58deeb789 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -899,6 +899,11 @@ class TDDnodes: dnodesRootDir = "%s/sim" % (self.path) return dnodesRootDir + def getDnodeDir(self, index): + self.check(index) + dnodesDir = "%s/sim/dnode%d" % (self.path, index) + return dnodesDir + def getSimCfgPath(self): return self.sim.getCfgDir() diff --git a/tests/army/test.py b/tests/army/test.py index a3e28b772d..37fecea370 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -405,11 +405,11 @@ if __name__ == "__main__": else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() for dnode in tdDnodes.dnodes: tdDnodes.deploy(dnode.index, updateCfgDict) for dnode in tdDnodes.dnodes: @@ -591,12 +591,11 @@ if __name__ == "__main__": print(independentMnode,"independentMnode valuse") # create dnode list dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.setAsan(asan) - tdDnodes.stopAll() + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() for dnode in tdDnodes.dnodes: tdDnodes.deploy(dnode.index,updateCfgDict) for dnode in tdDnodes.dnodes: