diff --git a/tests/army/community/cluster/incSnapshot.py b/tests/army/community/cluster/incSnapshot.py new file mode 100644 index 0000000000..b1b53e65bd --- /dev/null +++ b/tests/army/community/cluster/incSnapshot.py @@ -0,0 +1,105 @@ +import taos +import sys +import os +import subprocess +import glob +import shutil +import time + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame import * +from frame.autogen import * +from frame.server.dnodes import * +from frame.server.cluster import * + + +class TDTestCase(TBase): + + def init(self, conn, logSql, replicaVar=3): + super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1") + self.valgrind = 0 + self.childtable_count = 10 + # tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + tdSql.prepare() + autoGen = AutoGen() + autoGen.create_db(self.db, 2, 3) + tdSql.execute(f"use {self.db}") + autoGen.create_stable(self.stb, 5, 10, 8, 8) + autoGen.create_child(self.stb, "d", self.childtable_count) + autoGen.insert_data(1000) + tdSql.execute(f"flush database {self.db}") + clusterDnodes.stoptaosd(3) + # clusterDnodes.stoptaosd(1) + # clusterDnodes.starttaosd(3) + # time.sleep(5) + # clusterDnodes.stoptaosd(2) + # clusterDnodes.starttaosd(1) + # time.sleep(5) + autoGen.insert_data(5000, True) + tdSql.execute(f"flush database {self.db}") + + # sql = 'show vnodes;' + # while True: + # bFinish = True + # param_list = tdSql.query(sql, row_tag=True) + # for param in param_list: + # if param[3] == 'leading' or param[3] == 'following': + # bFinish = False + # break + # if bFinish: + # break + self.snapshotAgg() + time.sleep(10) + clusterDnodes.stopAll() + for i in range(1, 4): + path = clusterDnodes.getDnodeDir(i) + dnodesRootDir = os.path.join(path,"data","vnode", "vnode*") + dirs = glob.glob(dnodesRootDir) + for dir in dirs: + if os.path.isdir(dir): + tdLog.debug("delete dir: %s " % (dnodesRootDir)) + self.remove_directory(os.path.join(dir, "wal")) + + clusterDnodes.starttaosd(1) + clusterDnodes.starttaosd(2) + clusterDnodes.starttaosd(3) + sql = "show vnodes;" + time.sleep(10) + while True: + bFinish = True + param_list = tdSql.query(sql, row_tag=True) + for param in param_list: + if param[3] == 'offline': + tdLog.exit( + "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1])) + if param[3] == 'leading' or param[3] == 'following': + bFinish = False + break + if bFinish: + break + + self.timestamp_step = 1 + self.insert_rows = 6000 + self.checkInsertCorrect() + self.checkAggCorrect() + + def remove_directory(self, directory): + try: + shutil.rmtree(directory) + tdLog.debug("delete dir: %s " % (directory)) + except OSError as e: + tdLog.exit("delete fail dir: %s " % (directory)) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/autogen.py b/tests/army/frame/autogen.py index 1e041f633d..d1f02e7865 100644 --- a/tests/army/frame/autogen.py +++ b/tests/army/frame/autogen.py @@ -162,14 +162,19 @@ class AutoGen: tdLog.info(f" insert data i={i}") values = "" - tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}") + return ts - # insert data - def insert_data(self, cnt): + def insert_data(self, cnt, bContinue=False): + if not bContinue: + self.ts = 1600000000000 + + currTs = 1600000000000 for i in range(self.child_cnt): name = f"{self.child_name}{i}" - self.insert_data_child(name, cnt, self.batch_size, 1) + currTs = self.insert_data_child(name, cnt, self.batch_size, 1) + self.ts = currTs tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}") # insert same timestamp to all childs diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index f562ea1086..2959cf54a1 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -29,7 +29,7 @@ class TBase: # # init - def init(self, conn, logSql, replicaVar=1): + def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"): # save param self.replicaVar = int(replicaVar) tdSql.init(conn.cursor(), True) @@ -41,14 +41,14 @@ class TBase: self.mLevelDisk = 0 # test case information - self.db = "db" - self.stb = "stb" + self.db = db + self.stb = stb # sql - self.sqlSum = f"select sum(ic) from {self.stb}" - self.sqlMax = f"select max(ic) from {self.stb}" - self.sqlMin = f"select min(ic) from {self.stb}" - self.sqlAvg = f"select avg(ic) from {self.stb}" + self.sqlSum = f"select sum({checkColName}) from {self.stb}" + self.sqlMax = f"select max({checkColName}) from {self.stb}" + self.sqlMin = f"select min({checkColName}) from {self.stb}" + self.sqlAvg = f"select avg({checkColName}) from {self.stb}" self.sqlFirst = f"select first(ts) from {self.stb}" self.sqlLast = f"select last(ts) from {self.stb}" diff --git a/tests/army/frame/server/cluster.py b/tests/army/frame/server/cluster.py index aba2d2e630..309ca3bbd4 100644 --- a/tests/army/frame/server/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -13,23 +13,24 @@ from frame.common import * class ClusterDnodes(TDDnodes): """rewrite TDDnodes and make MyDdnodes as TDDnodes child class""" - def __init__(self ,dnodes_lists): - + def __init__(self): super(ClusterDnodes,self).__init__() - self.dnodes = dnodes_lists # dnode must be TDDnode instance self.simDeployed = False self.testCluster = False self.valgrind = 0 self.killValgrind = 1 + def init(self, dnodes_lists, deployPath, masterIp): + self.dnodes = dnodes_lists # dnode must be TDDnode instance + super(ClusterDnodes, self).init(deployPath, masterIp) +clusterDnodes = ClusterDnodes() class ConfigureyCluster: """This will create defined number of dnodes and create a cluster. at the same time, it will return TDDnodes list: dnodes, """ hostname = socket.gethostname() - def __init__(self): - self.dnodes = [] + self.dnodes = [] self.dnodeNums = 5 self.independent = True self.startPort = 6030 diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index a22e1cbff7..ad8a336857 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -251,6 +251,11 @@ class TDDnodes: dnodesRootDir = "%s/sim" % (self.path) return dnodesRootDir + def getDnodeDir(self, index): + self.check(index) + dnodesDir = "%s/sim/dnode%d" % (self.path, index) + return dnodesDir + def getSimCfgPath(self): return self.sim.getCfgDir() diff --git a/tests/army/test.py b/tests/army/test.py index a94fefe0be..5ff1c7bdf5 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -405,15 +405,15 @@ if __name__ == "__main__": else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.stopAll() - for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index, updateCfgDict) - for dnode in tdDnodes.dnodes: - tdDnodes.starttaosd(dnode.index) + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() + for dnode in clusterDnodes.dnodes: + clusterDnodes.deploy(dnode.index, updateCfgDict) + for dnode in clusterDnodes.dnodes: + clusterDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) if restful or websocket: @@ -580,16 +580,15 @@ if __name__ == "__main__": print(independentMnode,"independentMnode valuse") # create dnode list dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) - tdDnodes = ClusterDnodes(dnodeslist) - tdDnodes.init(deployPath, masterIp) - tdDnodes.setTestCluster(testCluster) - tdDnodes.setValgrind(valgrind) - tdDnodes.setAsan(asan) - tdDnodes.stopAll() - for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index,updateCfgDict) - for dnode in tdDnodes.dnodes: - tdDnodes.starttaosd(dnode.index) + clusterDnodes.init(dnodeslist, deployPath, masterIp) + clusterDnodes.setTestCluster(testCluster) + clusterDnodes.setValgrind(valgrind) + clusterDnodes.setAsan(asan) + clusterDnodes.stopAll() + for dnode in clusterDnodes.dnodes: + clusterDnodes.deploy(dnode.index,updateCfgDict) + for dnode in clusterDnodes.dnodes: + clusterDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) if restful or websocket: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 0687a3271c..891a83568a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -17,6 +17,7 @@ fi ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 ,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3 -L 3 -D 2 ,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3 ,,n,army,python3 ./test.py -f community/cmdline/fullopt.py