test:modify testcase of muti-mnode

tests/system-test/1-insert/insertWithMoreVgroup.py

@@ -264,7 +264,7 @@ class TDTestCase:
         speedCreate=count/spendTime
         tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate))
         return
-    # test case1 base 
+
     def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,):
         tdSql.execute("use %s"%dbname)
         tdSql.query("show stables")
@@ -275,7 +275,9 @@ class TDTestCase:
             tdSql.query("select count(*) from %s%d"%(stbname,i))
             tdSql.checkData(0,0,rowsPerSTable)
         return 
-        return
+     
+     
+    # test case1 base 
     def test_case1(self):
         #stableCount=threadNumbersCtb
         parameterDict = {'vgroups':        1,    \

tests/system-test/6-cluster/5dnode3mnodeDropInsert.py

@@ -72,20 +72,20 @@ class TDTestCase:
         self._async_raise(thread.ident, SystemExit)
 
 
-    def createDbTbale(self,countstart,countstop,count):
+    def createDbTbale(self,dbcountStart,dbcountStop,stbname,chilCount):
         # fisrt add data : db\stable\childtable\general table
         
-        for couti in range(countstart,countstop):
+        for couti in range(dbcountStart,dbcountStop):
             tdLog.debug("drop database if exists db%d" %couti)
             tdSql.execute("drop database if exists db%d" %couti)
             print("create database if not exists db%d replica 1 duration 300" %couti)
             tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
             tdSql.execute("use db%d" %couti)
             tdSql.execute(
-            '''create table stb1
+            '''create table %s
             (ts timestamp, c1 int, c2 bigint,c3 binary(16), c4 timestamp)
             tags (t1 int)
-            '''
+            '''%stbname
             )
             tdSql.execute(
                 '''
@@ -93,13 +93,13 @@ class TDTestCase:
                 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
                 '''
             )
-            for i in range(count):
+            for i in range(chilCount):
-                tdSql.execute(f'create table ct_{i+1} using stb1 tags ( {i+1} )')
+                tdSql.execute(f'create table {stbname}_{i+1} using {stbname} tags ( {i+1} )')
 
-    def insertTabaleData(self,countstart,countstop,stbname,chilCount,ts_start,rowCount):
+    def insertTabaleData(self,dbcountStart,dbcountStop,stbname,chilCount,ts_start,rowCount):
         #  insert data : create childtable and data
         
-        for couti in range(countstart,countstop):
+        for couti in range(dbcountStart,dbcountStop):
             tdSql.execute("use db%d" %couti)
             pre_insert = "insert into "
             sql = pre_insert
@@ -125,6 +125,18 @@ class TDTestCase:
             speedInsert=allRows/spendTime
             tdLog.debug("spent %.2fs to INSERT  %d rows into %s , insert rate is  %.2f rows/s... [OK]"% (spendTime,allRows,stbname,speedInsert))
 
+    def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,):
+        tdSql.execute("use %s"%dbname)
+        tdSql.query("show stables")
+        tdSql.checkRows(stableCount)
+        tdSql.query("show tables")
+        tdSql.checkRows(CtableCount)
+        for i in range(stableCount):
+            tdSql.query("select count(*) from %s%d"%(stbname,i))
+            tdSql.checkData(0,0,rowsPerSTable)
+        return 
+     
+
     def depoly_cluster(self ,dnodes_nums): 
 
         testCluster = False
@@ -321,6 +333,16 @@ class TDTestCase:
         tdSql.checkData(2,3,'ready')
 
     def five_dnode_three_mnode(self,dnodenumber):
+        # testcase parameters
+        vgroups=1
+        dbcountStart=0
+        dbcountStop=1
+        dbname="db"
+        stbname="stb"
+        tablesPerStb=1000
+        rowsPerTable=100
+        startTs=1640966400000  # 2022-01-01 00:00:00.000
+
         tdSql.query("show dnodes;")
         tdSql.checkData(0,1,'%s:6030'%self.host)
         tdSql.checkData(4,1,'%s:6430'%self.host)
@@ -345,10 +367,16 @@ class TDTestCase:
         tdLog.debug("stop all of mnode ") 
 
         #  drop follower of mnode and insert data
-        self.createDbTbale(0,1,1000)
+        self.createDbTbale(dbcountStart, dbcountStop,stbname,tablesPerStb)
-#method) insertTabaleData: (countstart: Any, countstop: Any, stbname: Any, chilCount: Any, ts_start: Any, rowCount: Any) -> None
+        #(method) insertTabaleData: (dbcountStart: Any, dbcountStop: Any, stbname: Any, chilCount: Any, ts_start: Any, rowCount: Any) -> None
+        threads=threading.Thread(target=self.insertTabaleData, args=(
+        dbcountStart,
+        dbcountStop,
+        stbname,
+        tablesPerStb,
+        startTs,
+        rowsPerTable))
 
-        threads=threading.Thread(target=self.insertTabaleData, args=(0,1,"ct",1000,self.ts,100))
         threads.start()
         dropcount =0 
         while dropcount <= 10:
@@ -379,6 +407,7 @@ class TDTestCase:
         self.check3mnode()
 
 
+
     def getConnection(self, dnode):
         host = dnode.cfgDict["fqdn"]
         port = dnode.cfgDict["serverPort"]

tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py

@@ -0,0 +1,377 @@
+from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE
+import taos
+import sys
+import time
+import os 
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import TDDnodes
+from util.dnodes import TDDnode
+import time
+import socket
+import subprocess
+from multiprocessing import Process
+import threading 
+import time
+import inspect
+import ctypes
+class MyDnodes(TDDnodes):
+    def __init__(self ,dnodes_lists):
+        super(MyDnodes,self).__init__()
+        self.dnodes = dnodes_lists  # dnode must be TDDnode instance
+        self.simDeployed = False
+
+
+class TDTestCase:
+
+    def init(self,conn ,logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        self.TDDnodes = None
+
+    def buildcluster(self,dnodenumber):
+        self.depoly_cluster(dnodenumber)
+        self.master_dnode = self.TDDnodes.dnodes[0]
+        self.host=self.master_dnode.cfgDict["fqdn"]
+        conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir)
+        tdSql.init(conn1.cursor())
+        
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def _async_raise(self, tid, exctype):
+        """raises the exception, performs cleanup if needed"""
+        if not inspect.isclass(exctype):
+            exctype = type(exctype)
+        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
+        if res == 0:
+            raise ValueError("invalid thread id")
+        elif res != 1:
+            # """if it returns a number greater than one, you're in trouble, 
+            # and you should call it again with exc=NULL to revert the effect"""
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
+            raise SystemError("PyThreadState_SetAsyncExc failed")
+
+    def stop_thread(self,thread):
+        self._async_raise(thread.ident, SystemExit)
+
+
+    def insert_data(self,countstart,countstop):
+        # fisrt add data : db\stable\childtable\general table
+        
+        for couti in range(countstart,countstop):
+            tdLog.debug("drop database if exists db%d" %couti)
+            tdSql.execute("drop database if exists db%d" %couti)
+            print("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti)
+            tdSql.execute("use db%d" %couti)
+            tdSql.execute(
+            '''create table stb1
+            (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+            tags (t1 int)
+            '''
+            )
+            tdSql.execute(
+                '''
+                create table t1
+                (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
+                '''
+            )
+            for i in range(4):
+                tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )')
+
+    def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,):
+        tdSql.execute("use %s"%dbname)
+        tdSql.query("show stables")
+        tdSql.checkRows(stableCount)
+        tdSql.query("show tables")
+        tdSql.checkRows(CtableCount)
+        for i in range(stableCount):
+            tdSql.query("select count(*) from %s%d"%(stbname,i))
+            tdSql.checkData(0,0,rowsPerSTable)
+        return 
+
+    def depoly_cluster(self ,dnodes_nums=5,independent=True): 
+
+        testCluster = False
+        valgrind = 0  
+        hostname = socket.gethostname()
+        dnodes = []
+        start_port = 6030
+        start_port_sec = 6130
+        for num in range(1, dnodes_nums+1):
+            dnode = TDDnode(num)
+            dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}")
+            dnode.addExtraCfg("fqdn", f"{hostname}")
+            dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}")
+            dnode.addExtraCfg("monitorFqdn", hostname)
+            dnode.addExtraCfg("monitorPort", 7043)
+            dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}")
+            # configure three dnoe don't support vnodes
+            if independent and (num < 4):
+                dnode.addExtraCfg("supportVnodes", 0)
+
+            dnodes.append(dnode)
+        
+        self.TDDnodes = MyDnodes(dnodes)
+        self.TDDnodes.init("")
+        self.TDDnodes.setTestCluster(testCluster)
+        self.TDDnodes.setValgrind(valgrind)
+        self.TDDnodes.stopAll()
+        for dnode in self.TDDnodes.dnodes:
+            self.TDDnodes.deploy(dnode.index,{})
+            
+        for dnode in self.TDDnodes.dnodes:
+            self.TDDnodes.starttaosd(dnode.index)
+
+        # create cluster 
+        for dnode in self.TDDnodes.dnodes[1:]:
+            # print(dnode.cfgDict)
+            dnode_id = dnode.cfgDict["fqdn"] +  ":" +dnode.cfgDict["serverPort"]
+            dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0]
+            dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1]
+            cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;"
+            print(cmd)
+            os.system(cmd)
+        
+        time.sleep(2)
+        tdLog.info(" create cluster with %d dnode  done! " %dnodes_nums)
+
+    def checkdnodes(self,dnodenumber):
+        count=0
+        while count < 100:
+            time.sleep(1)
+            statusReadyBumber=0
+            tdSql.query("show dnodes;")
+            if tdSql.checkRows(dnodenumber) :     
+                print("dnode is %d nodes"%dnodenumber)   
+            for i in range(dnodenumber):
+                if tdSql.queryResult[i][4] !='ready'  :
+                    status=tdSql.queryResult[i][4]
+                    print("dnode:%d status is %s "%(i,status))
+                    break
+                else:
+                    statusReadyBumber+=1
+            print(statusReadyBumber)
+            if statusReadyBumber == dnodenumber :
+                print("all of %d mnodes is ready in 10s "%dnodenumber)
+                return True
+                break
+            count+=1
+        else:
+            print("%d mnodes is not ready in 10s "%dnodenumber)
+            return False
+           
+
+    def check3mnode(self):
+        count=0
+        while count < 10:
+            time.sleep(1)
+            tdSql.query("show mnodes;")
+            if tdSql.checkRows(3) :     
+                print("mnode is  three nodes")           
+            if  tdSql.queryResult[0][2]=='leader' :
+                if  tdSql.queryResult[1][2]=='follower':
+                    if  tdSql.queryResult[2][2]=='follower':
+                        print("three mnodes is ready in 10s")
+                        break
+            elif tdSql.queryResult[0][2]=='follower' :
+                if  tdSql.queryResult[1][2]=='leader':
+                    if  tdSql.queryResult[2][2]=='follower':
+                        print("three mnodes is ready in 10s")
+                        break      
+            elif tdSql.queryResult[0][2]=='follower' :
+                if  tdSql.queryResult[1][2]=='follower':
+                    if  tdSql.queryResult[2][2]=='leader':
+                        print("three mnodes is ready in 10s")
+                        break                   
+            count+=1
+        else:
+            print("three mnodes is not ready in 10s ")
+            return -1
+
+        tdSql.query("show mnodes;")       
+        tdSql.checkRows(3) 
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(0,3,'ready')
+        tdSql.checkData(1,1,'%s:6130'%self.host)
+        tdSql.checkData(1,3,'ready')
+        tdSql.checkData(2,1,'%s:6230'%self.host)
+        tdSql.checkData(2,3,'ready')
+
+    def check3mnode1off(self):
+        count=0
+        while count < 10:
+            time.sleep(1)
+            tdSql.query("show mnodes;")
+            if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
+            if  tdSql.queryResult[0][2]=='offline' :
+                if  tdSql.queryResult[1][2]=='leader':
+                    if  tdSql.queryResult[2][2]=='follower':
+                        print("stop mnodes  on dnode 2 successfully in 10s")
+                        break
+                elif tdSql.queryResult[1][2]=='follower':
+                    if  tdSql.queryResult[2][2]=='leader':
+                        print("stop mnodes  on dnode 2 successfully in 10s")
+                        break
+            count+=1
+        else:
+            print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1
+        tdSql.error("drop mnode on dnode 1;")
+
+        tdSql.query("show mnodes;")       
+        tdSql.checkRows(3) 
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(0,2,'offline')
+        tdSql.checkData(0,3,'ready')
+        tdSql.checkData(1,1,'%s:6130'%self.host)
+        tdSql.checkData(1,3,'ready')
+        tdSql.checkData(2,1,'%s:6230'%self.host)
+        tdSql.checkData(2,3,'ready')
+
+    def check3mnode2off(self):
+        count=0
+        while count < 40:
+            time.sleep(1)
+            tdSql.query("show mnodes;")
+            if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
+            if  tdSql.queryResult[0][2]=='leader' :
+                if  tdSql.queryResult[1][2]=='offline':
+                    if  tdSql.queryResult[2][2]=='follower':
+                        print("stop mnodes  on dnode 2 successfully in 10s")
+                        break
+            count+=1
+        else:
+            print("stop mnodes  on dnode 2 failed in 10s ")
+            return -1
+        tdSql.error("drop mnode on dnode 2;")
+
+        tdSql.query("show mnodes;")       
+        tdSql.checkRows(3) 
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(0,2,'leader')
+        tdSql.checkData(0,3,'ready')
+        tdSql.checkData(1,1,'%s:6130'%self.host)
+        tdSql.checkData(1,2,'offline')
+        tdSql.checkData(1,3,'ready')
+        tdSql.checkData(2,1,'%s:6230'%self.host)
+        tdSql.checkData(2,2,'follower')
+        tdSql.checkData(2,3,'ready')
+
+    def check3mnode3off(self):
+        count=0
+        while count < 10:
+            time.sleep(1)
+            tdSql.query("show mnodes;")
+            if tdSql.checkRows(3) :
+                print("mnode is  three nodes")
+            if  tdSql.queryResult[0][2]=='leader' :
+                if  tdSql.queryResult[2][2]=='offline':
+                    if  tdSql.queryResult[1][2]=='follower':
+                        print("stop mnodes  on dnode 3 successfully in 10s")
+                        break
+            count+=1
+        else:
+            print("stop mnodes  on dnode 3 failed in 10s")
+            return -1
+        tdSql.error("drop mnode on dnode 3;")
+        tdSql.query("show mnodes;")       
+        tdSql.checkRows(3) 
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(0,2,'leader')
+        tdSql.checkData(0,3,'ready')
+        tdSql.checkData(1,1,'%s:6130'%self.host)
+        tdSql.checkData(1,2,'follower')
+        tdSql.checkData(1,3,'ready')
+        tdSql.checkData(2,1,'%s:6230'%self.host)
+        tdSql.checkData(2,2,'offline')
+        tdSql.checkData(2,3,'ready')
+
+    def five_dnode_three_mnode(self,dnodenumber):
+        tdSql.query("show dnodes;")
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(4,1,'%s:6430'%self.host)
+        tdSql.checkData(0,4,'ready')
+        tdSql.checkData(4,4,'ready')
+        tdSql.query("show mnodes;")   
+        tdSql.checkRows(1)    
+        tdSql.checkData(0,1,'%s:6030'%self.host)
+        tdSql.checkData(0,2,'leader')
+        tdSql.checkData(0,3,'ready')
+
+        # fisr add three mnodes;
+        tdSql.execute("create mnode on dnode 2")
+        tdSql.execute("create mnode on dnode 3")
+
+        # fisrt check statut ready
+        self.check3mnode()
+
+        tdSql.error("create mnode on dnode 2")
+        tdSql.query("show dnodes;")
+        print(tdSql.queryResult)
+        tdLog.debug("stop all of mnode ") 
+
+        # seperate vnode and mnode in different dnodes.
+        # create database and stable
+        stopcount =0 
+        while stopcount < 2:
+            for i in range(dnodenumber):
+                # threads=[]
+                # threads = MyThreadFunc(self.insert_data(i*2,i*2+2)) 
+                threads=threading.Thread(target=self.insert_data, args=(i,i+1))
+                threads.start()
+                self.TDDnodes.stoptaosd(i+1)
+                self.TDDnodes.starttaosd(i+1)
+
+                if self.checkdnodes(5):
+                    print("123")
+                    threads.join()
+                else:
+                    print("456")
+                    self.stop_thread(threads)
+                    assert 1 == 2 ,"some dnode started failed"
+                    return False
+                # self.check3mnode()
+            self.check3mnode()
+            
+
+            stopcount+=1
+        self.check3mnode()
+
+
+    def getConnection(self, dnode):
+        host = dnode.cfgDict["fqdn"]
+        port = dnode.cfgDict["serverPort"]
+        config_dir = dnode.cfgDir
+        return taos.connect(host=host, port=int(port), config=config_dir)
+
+
+    def run(self): 
+        # print(self.master_dnode.cfgDict)
+        self.buildcluster(5)
+        self.five_dnode_three_mnode(5)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())