add inc Snapshot copy case

tests/army/community/cluster/incSnapshot.py

@@ -0,0 +1,104 @@
+import taos
+import sys
+import os
+import subprocess
+import glob
+import shutil
+import time
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+from frame.server.dnodes import *
+from frame.server.cluster import *
+
+
+class TDTestCase(TBase):
+
+    def init(self, conn, logSql, replicaVar=3):
+        super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1")
+        self.valgrind = 0
+        self.childtable_count = 10
+        # tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def run(self):
+        tdSql.prepare()
+        autoGen = AutoGen()
+        autoGen.create_db(self.db, 2, 3)
+        autoGen.create_stable(self.stb, 5, 10, 8, 8)
+        autoGen.create_child(self.stb, "d", self.childtable_count)
+        autoGen.insert_data(1)
+        tdSql.execute(f"flush database {self.db}")
+        clusterDnodes.stoptaosd(3)
+        clusterDnodes.stoptaosd(1)
+        clusterDnodes.starttaosd(3)
+        time.sleep(5)
+        clusterDnodes.stoptaosd(2)
+        clusterDnodes.starttaosd(1)
+        time.sleep(5)
+        autoGen.insert_data(10, 1600000000001)
+        # tdSql.execute(f"flush database {self.db}")
+
+        sql = 'show vnodes;'
+        while True:
+            bFinish = True
+            param_list = tdSql.query(sql, row_tag=True)
+            for param in param_list:
+                if param[3] == 'leading' or param[3] == 'following':
+                    bFinish = False
+                    break
+            if bFinish:
+                break
+        self.snapshotAgg()
+
+        clusterDnodes.stopAll()
+        # for i in range(1, 4):
+        #     path = clusterDnodes.getDnodeDir(i)
+        #     dnodesRootDir = os.path.join(path,"data","vnode", "vnode*")
+        #     dirs = glob.glob(dnodesRootDir)
+        #     for dir in dirs:
+        #         if os.path.isdir(dir):
+        #             tdLog.debug("delete dir: %s " % (dnodesRootDir))
+        #             self.remove_directory(os.path.join(dir, "wal"))
+
+        clusterDnodes.starttaosd(1)
+        clusterDnodes.starttaosd(2)
+        clusterDnodes.starttaosd(3)
+
+        time.sleep(3)
+        while True:
+            bFinish = True
+            param_list = tdSql.query(sql, row_tag=True)
+            for param in param_list:
+                if param[3] == 'offline':
+                    tdLog.exit(
+                        "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1]))
+                if param[3] == 'leading' or param[3] == 'following':
+                    bFinish = False
+                    break
+            if bFinish:
+                break
+
+        self.timestamp_step = 1
+        self.insert_rows = 11
+        self.checkInsertCorrect()
+        self.checkAggCorrect()
+
+    def remove_directory(self, directory):
+        try:
+            shutil.rmtree(directory)
+            tdLog.debug("delete dir: %s " % (directory))
+        except OSError as e:
+            tdLog.exit("delete fail dir: %s " % (directory))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/army/frame/autogen.py

@@ -138,18 +138,25 @@ class AutoGen:
         # end batch
         if values != "":
             sql = f"insert into {child_name} values {values}"
+            tdLog.info(f" insert child data SQL{sql}")
             tdSql.execute(sql)
             tdLog.info(f" insert data i={i}")
             values = ""
 
-        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")    
+        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt} last_ts={ts}")
+        return ts
 
     # insert data
-    def insert_data(self, cnt):
+    def insert_data(self, cnt, bContinue=False):
+        if not bContinue:
+            self.ts = 1600000000000
+
+        currTs = 1600000000000
         for i in range(self.child_cnt):
             name = f"{self.child_name}{i}"
-            self.insert_data_child(name, cnt, self.batch_size, 1)
+            currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
 
+        self.ts = currTs
         tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
 
     # insert same timestamp to all childs
@@ -159,5 +166,3 @@ class AutoGen:
             self.insert_data_child(name, cnt, self.batch_size, 0)
 
         tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")
-
-

tests/army/frame/caseBase.py

@@ -28,7 +28,7 @@ class TBase:
 #          
 
     # init
-    def init(self, conn, logSql, replicaVar=1):
+    def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"):
         # save param
         self.replicaVar = int(replicaVar)
         tdSql.init(conn.cursor(), True)
@@ -40,14 +40,14 @@ class TBase:
         self.mLevelDisk = 0
 
         # test case information
-        self.db     = "db"
+        self.db     = db
-        self.stb    = "stb"
+        self.stb    = stb
 
         # sql 
-        self.sqlSum = f"select sum(ic) from {self.stb}"
+        self.sqlSum = f"select sum({checkColName}) from {self.stb}"
-        self.sqlMax = f"select max(ic) from {self.stb}"
+        self.sqlMax = f"select max({checkColName}) from {self.stb}"
-        self.sqlMin = f"select min(ic) from {self.stb}"
+        self.sqlMin = f"select min({checkColName}) from {self.stb}"
-        self.sqlAvg = f"select avg(ic) from {self.stb}"
+        self.sqlAvg = f"select avg({checkColName}) from {self.stb}"
         self.sqlFirst = f"select first(ts) from {self.stb}"
         self.sqlLast  = f"select last(ts) from {self.stb}"
 

tests/army/frame/server/cluster.py

@@ -13,21 +13,22 @@ from frame.common import *
 
 class ClusterDnodes(TDDnodes):
     """rewrite TDDnodes and make MyDdnodes as TDDnodes child class"""
-    def __init__(self ,dnodes_lists):
+    def __init__(self):
-
         super(ClusterDnodes,self).__init__()
-        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
         self.simDeployed = False
         self.testCluster = False
         self.valgrind = 0
         self.killValgrind = 1
+    def init(self, dnodes_lists, deployPath, masterIp):
+        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
+        super(ClusterDnodes, self).init(deployPath, masterIp)
 
+clusterDnodes = ClusterDnodes()
 
 class ConfigureyCluster:
     """This will create defined number of dnodes and create a cluster.
        at the same time, it will return TDDnodes list:  dnodes, """
     hostname = socket.gethostname()
-
     def __init__(self):
         self.dnodes = []
         self.dnodeNums = 5

tests/army/frame/server/dnodes.py

@@ -899,6 +899,11 @@ class TDDnodes:
         dnodesRootDir = "%s/sim" % (self.path)
         return dnodesRootDir
 
+    def getDnodeDir(self, index):
+        self.check(index)
+        dnodesDir = "%s/sim/dnode%d" % (self.path, index)
+        return dnodesDir
+
     def getSimCfgPath(self):
         return self.sim.getCfgDir()
 

tests/army/test.py

@@ -405,11 +405,11 @@ if __name__ == "__main__":
         else :
             tdLog.debug("create an cluster  with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
-            tdDnodes = ClusterDnodes(dnodeslist)
+            clusterDnodes.init(dnodeslist, deployPath, masterIp)
-            tdDnodes.init(deployPath, masterIp)
+            clusterDnodes.setTestCluster(testCluster)
-            tdDnodes.setTestCluster(testCluster)
+            clusterDnodes.setValgrind(valgrind)
-            tdDnodes.setValgrind(valgrind)
+            clusterDnodes.setAsan(asan)
-            tdDnodes.stopAll()
+            clusterDnodes.stopAll()
             for dnode in tdDnodes.dnodes:
                 tdDnodes.deploy(dnode.index, updateCfgDict)
             for dnode in tdDnodes.dnodes:
@@ -591,12 +591,11 @@ if __name__ == "__main__":
             print(independentMnode,"independentMnode valuse")
             # create dnode list
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
-            tdDnodes = ClusterDnodes(dnodeslist)
+            clusterDnodes.init(dnodeslist, deployPath, masterIp)
-            tdDnodes.init(deployPath, masterIp)
+            clusterDnodes.setTestCluster(testCluster)
-            tdDnodes.setTestCluster(testCluster)
+            clusterDnodes.setValgrind(valgrind)
-            tdDnodes.setValgrind(valgrind)
+            clusterDnodes.setAsan(asan)
-            tdDnodes.setAsan(asan)
+            clusterDnodes.stopAll()
-            tdDnodes.stopAll()
             for dnode in tdDnodes.dnodes:
                 tdDnodes.deploy(dnode.index,updateCfgDict)
             for dnode in tdDnodes.dnodes: