add inc Snapshot copy case

tests/army/community/cluster/incSnapshot.py

@@ -0,0 +1,104 @@
+import taos
+import sys
+import os
+import subprocess
+import glob
+import shutil
+import time
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+from frame.server.dnodes import *
+from frame.server.cluster import *
+
+
+class TDTestCase(TBase):
+
+    def init(self, conn, logSql, replicaVar=3):
+        super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1")
+        self.valgrind = 0
+        self.childtable_count = 10
+        # tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def run(self):
+        tdSql.prepare()
+        autoGen = AutoGen()
+        autoGen.create_db(self.db, 2, 3)
+        autoGen.create_stable(self.stb, 5, 10, 8, 8)
+        autoGen.create_child(self.stb, "d", self.childtable_count)
+        autoGen.insert_data(1)
+        tdSql.execute(f"flush database {self.db}")
+        clusterDnodes.stoptaosd(3)
+        clusterDnodes.stoptaosd(1)
+        clusterDnodes.starttaosd(3)
+        time.sleep(5)
+        clusterDnodes.stoptaosd(2)
+        clusterDnodes.starttaosd(1)
+        time.sleep(5)
+        autoGen.insert_data(10, 1600000000001)
+        # tdSql.execute(f"flush database {self.db}")
+
+        sql = 'show vnodes;'
+        while True:
+            bFinish = True
+            param_list = tdSql.query(sql, row_tag=True)
+            for param in param_list:
+                if param[3] == 'leading' or param[3] == 'following':
+                    bFinish = False
+                    break
+            if bFinish:
+                break
+        self.snapshotAgg()
+
+        clusterDnodes.stopAll()
+        # for i in range(1, 4):
+        #     path = clusterDnodes.getDnodeDir(i)
+        #     dnodesRootDir = os.path.join(path,"data","vnode", "vnode*")
+        #     dirs = glob.glob(dnodesRootDir)
+        #     for dir in dirs:
+        #         if os.path.isdir(dir):
+        #             tdLog.debug("delete dir: %s " % (dnodesRootDir))
+        #             self.remove_directory(os.path.join(dir, "wal"))
+
+        clusterDnodes.starttaosd(1)
+        clusterDnodes.starttaosd(2)
+        clusterDnodes.starttaosd(3)
+
+        time.sleep(3)
+        while True:
+            bFinish = True
+            param_list = tdSql.query(sql, row_tag=True)
+            for param in param_list:
+                if param[3] == 'offline':
+                    tdLog.exit(
+                        "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1]))
+                if param[3] == 'leading' or param[3] == 'following':
+                    bFinish = False
+                    break
+            if bFinish:
+                break
+
+        self.timestamp_step = 1
+        self.insert_rows = 11
+        self.checkInsertCorrect()
+        self.checkAggCorrect()
+
+    def remove_directory(self, directory):
+        try:
+            shutil.rmtree(directory)
+            tdLog.debug("delete dir: %s " % (directory))
+        except OSError as e:
+            tdLog.exit("delete fail dir: %s " % (directory))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/army/frame/autogen.py

@@ -30,7 +30,7 @@ class AutoGen:
 
     #  _columns_sql
     def gen_columns_sql(self, pre, cnt, binary_len, nchar_len):
-        types = [ 
+        types = [
             'timestamp',
             'tinyint',
             'smallint',
@@ -58,33 +58,33 @@ class AutoGen:
                 sqls += ","
             sqls += sql
             metas.append(sel)
-        
+
-        return metas, sqls;    
+        return metas, sqls;
 
     # gen tags data
     def gen_data(self, i, marr):
-        datas = ""   
+        datas = ""
         for c in marr:
             data = ""
-            if c == 0 : # timestamp
+            if c == 0:  # timestamp
                 data = "%d" % (self.ts + i)
-            elif c <= 4 : # small
+            elif c <= 4:  # small
-                data = "%d"%(i%128)
+                data = "%d" % (i % 128)
-            elif c <= 8 : # int
+            elif c <= 8:  # int
                 data = f"{i}"
-            elif c <= 10 : # float
+            elif c <= 10:  # float
-                data = "%f"%(i+i/1000)
+                data = "%f" % (i + i / 1000)
-            elif c <= 11 : # bool
+            elif c <= 11:  # bool
-                data = "%d"%(i%2)
+                data = "%d" % (i % 2)
-            elif c == 12 : # binary
+            elif c == 12:  # binary
                 data = '"' + self.random_string(self.bin_len) + '"'
-            elif c == 13 : # binary
+            elif c == 13:  # binary
                 data = '"' + self.random_string(self.nch_len) + '"'
 
             if datas != "":
                 datas += ","
             datas += data
-        
+
         return datas
 
     # generate specail wide random string
@@ -93,11 +93,11 @@ class AutoGen:
         return ''.join(random.choice(letters) for i in range(count))
 
     # create db
-    def create_db(self, dbname, vgroups = 2, replica = 1):
+    def create_db(self, dbname, vgroups=2, replica=1):
-        self.dbname  = dbname
+        self.dbname = dbname
         tdSql.execute(f'create database {dbname} vgroups {vgroups} replica {replica}')
         tdSql.execute(f'use {dbname}')
-        
+
     # create table or stable
     def create_stable(self, stbname, tag_cnt, column_cnt, binary_len, nchar_len):
         self.bin_len = binary_len
@@ -109,7 +109,7 @@ class AutoGen:
         sql = f"create table {stbname} (ts timestamp, {cols}) tags({tags})"
         tdSql.execute(sql)
 
-    # create child table 
+    # create child table
     def create_child(self, stbname, prename, cnt):
         self.child_cnt = cnt
         self.child_name = prename
@@ -120,7 +120,7 @@ class AutoGen:
 
         tdLog.info(f"create child tables {cnt} ok")
 
-    def insert_data_child(self, child_name, cnt, batch_size, step):        
+    def insert_data_child(self, child_name, cnt, batch_size, step):
         values = ""
         print("insert child data")
         ts = self.ts
@@ -130,7 +130,7 @@ class AutoGen:
             value = self.gen_data(i, self.mcols)
             ts += step
             values += f"({ts},{value}) "
-            if batch_size == 1 or (i > 0 and i % batch_size == 0) :
+            if batch_size == 1 or (i > 0 and i % batch_size == 0):
                 sql = f"insert into {child_name} values {values}"
                 tdSql.execute(sql)
                 values = ""
@@ -138,18 +138,25 @@ class AutoGen:
         # end batch
         if values != "":
             sql = f"insert into {child_name} values {values}"
+            tdLog.info(f" insert child data SQL{sql}")
             tdSql.execute(sql)
             tdLog.info(f" insert data i={i}")
             values = ""
 
-        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt}")    
+        tdLog.info(f" insert child data {child_name} finished, insert rows={cnt} last_ts={ts}")
+        return ts
 
-    # insert data 
+    # insert data
-    def insert_data(self, cnt):
+    def insert_data(self, cnt, bContinue=False):
+        if not bContinue:
+            self.ts = 1600000000000
+
+        currTs = 1600000000000
         for i in range(self.child_cnt):
             name = f"{self.child_name}{i}"
-            self.insert_data_child(name, cnt, self.batch_size, 1)
+            currTs = self.insert_data_child(name, cnt, self.batch_size, 1)
 
+        self.ts = currTs
         tdLog.info(f" insert data ok, child table={self.child_cnt} insert rows={cnt}")
 
     # insert same timestamp to all childs
@@ -158,6 +165,4 @@ class AutoGen:
             name = f"{self.child_name}{i}"
             self.insert_data_child(name, cnt, self.batch_size, 0)
 
-        tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")         
+        tdLog.info(f" insert same timestamp ok, child table={self.child_cnt} insert rows={cnt}")
-
-

tests/army/frame/caseBase.py

@@ -28,7 +28,7 @@ class TBase:
 #          
 
     # init
-    def init(self, conn, logSql, replicaVar=1):
+    def init(self, conn, logSql, replicaVar=1, db="db", stb="stb", checkColName="ic"):
         # save param
         self.replicaVar = int(replicaVar)
         tdSql.init(conn.cursor(), True)
@@ -40,14 +40,14 @@ class TBase:
         self.mLevelDisk = 0
 
         # test case information
-        self.db     = "db"
+        self.db     = db
-        self.stb    = "stb"
+        self.stb    = stb
 
         # sql 
-        self.sqlSum = f"select sum(ic) from {self.stb}"
+        self.sqlSum = f"select sum({checkColName}) from {self.stb}"
-        self.sqlMax = f"select max(ic) from {self.stb}"
+        self.sqlMax = f"select max({checkColName}) from {self.stb}"
-        self.sqlMin = f"select min(ic) from {self.stb}"
+        self.sqlMin = f"select min({checkColName}) from {self.stb}"
-        self.sqlAvg = f"select avg(ic) from {self.stb}"
+        self.sqlAvg = f"select avg({checkColName}) from {self.stb}"
         self.sqlFirst = f"select first(ts) from {self.stb}"
         self.sqlLast  = f"select last(ts) from {self.stb}"
 

tests/army/frame/server/cluster.py

@@ -13,23 +13,24 @@ from frame.common import *
 
 class ClusterDnodes(TDDnodes):
     """rewrite TDDnodes and make MyDdnodes as TDDnodes child class"""
-    def __init__(self ,dnodes_lists):
+    def __init__(self):
-
         super(ClusterDnodes,self).__init__()
-        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
         self.simDeployed = False
         self.testCluster = False
         self.valgrind = 0
         self.killValgrind = 1
+    def init(self, dnodes_lists, deployPath, masterIp):
+        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
+        super(ClusterDnodes, self).init(deployPath, masterIp)
 
+clusterDnodes = ClusterDnodes()
 
 class ConfigureyCluster:
     """This will create defined number of dnodes and create a cluster.
        at the same time, it will return TDDnodes list:  dnodes, """
     hostname = socket.gethostname()
-
     def __init__(self):
-        self.dnodes = []      
+        self.dnodes = []
         self.dnodeNums = 5
         self.independent = True
         self.startPort = 6030

tests/army/frame/server/dnodes.py

@@ -899,6 +899,11 @@ class TDDnodes:
         dnodesRootDir = "%s/sim" % (self.path)
         return dnodesRootDir
 
+    def getDnodeDir(self, index):
+        self.check(index)
+        dnodesDir = "%s/sim/dnode%d" % (self.path, index)
+        return dnodesDir
+
     def getSimCfgPath(self):
         return self.sim.getCfgDir()
 

tests/army/test.py

@@ -405,11 +405,11 @@ if __name__ == "__main__":
         else :
             tdLog.debug("create an cluster  with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums))
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
-            tdDnodes = ClusterDnodes(dnodeslist)
+            clusterDnodes.init(dnodeslist, deployPath, masterIp)
-            tdDnodes.init(deployPath, masterIp)
+            clusterDnodes.setTestCluster(testCluster)
-            tdDnodes.setTestCluster(testCluster)
+            clusterDnodes.setValgrind(valgrind)
-            tdDnodes.setValgrind(valgrind)
+            clusterDnodes.setAsan(asan)
-            tdDnodes.stopAll()
+            clusterDnodes.stopAll()
             for dnode in tdDnodes.dnodes:
                 tdDnodes.deploy(dnode.index, updateCfgDict)
             for dnode in tdDnodes.dnodes:
@@ -591,12 +591,11 @@ if __name__ == "__main__":
             print(independentMnode,"independentMnode valuse")
             # create dnode list
             dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk)
-            tdDnodes = ClusterDnodes(dnodeslist)
+            clusterDnodes.init(dnodeslist, deployPath, masterIp)
-            tdDnodes.init(deployPath, masterIp)
+            clusterDnodes.setTestCluster(testCluster)
-            tdDnodes.setTestCluster(testCluster)
+            clusterDnodes.setValgrind(valgrind)
-            tdDnodes.setValgrind(valgrind)
+            clusterDnodes.setAsan(asan)
-            tdDnodes.setAsan(asan)
+            clusterDnodes.stopAll()
-            tdDnodes.stopAll()
             for dnode in tdDnodes.dnodes:
                 tdDnodes.deploy(dnode.index,updateCfgDict)
             for dnode in tdDnodes.dnodes: