Merge pull request #24294 from taosdata/coverage/TD-28114-3.0

coverage: add snapshot.json case

tests/army/community/cluster/snapshot.json

@@ -0,0 +1,60 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "connection_pool_size": 8,
+    "num_of_records_per_req": 2000,
+    "thread_count": 2,
+    "create_table_thread_count": 1,
+    "confirm_parameter_prompt": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "db",
+                "drop": "yes",
+                "vgroups": 2,
+                "replica": 3,
+                "duration":"1d",
+                "wal_retention_period": 1,
+                "wal_retention_size": 1,
+                "keep": "3d,6d,30d"
+            },
+            "super_tables": [
+                {
+                    "name": "stb",
+                    "child_table_exists": "no",
+                    "childtable_count": 10,
+                    "insert_rows": 100000,
+                    "childtable_prefix": "d",
+                    "insert_mode": "taosc",
+                    "timestamp_step": 10000,
+                    "start_timestamp":"now-12d",
+                    "columns": [
+                        { "type": "bool",        "name": "bc"},
+                        { "type": "float",       "name": "fc" },
+                        { "type": "double",      "name": "dc"},
+                        { "type": "tinyint",     "name": "ti"},
+                        { "type": "smallint",    "name": "si" },
+                        { "type": "int",         "name": "ic" },
+                        { "type": "bigint",      "name": "bi" },
+                        { "type": "utinyint",    "name": "uti"},
+                        { "type": "usmallint",   "name": "usi"},
+                        { "type": "uint",        "name": "ui" },
+                        { "type": "ubigint",     "name": "ubi"},
+                        { "type": "binary",      "name": "bin", "len": 16},
+                        { "type": "nchar",       "name": "nch", "len": 32}
+                    ],
+                    "tags": [
+                        {"type": "tinyint", "name": "groupid","max": 10,"min": 1},
+                        {"name": "location","type": "binary", "len": 16, "values":
+                           ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}

tests/army/community/cluster/snapshot.py

@@ -0,0 +1,96 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+import random
+
+import taos
+import frame
+import frame.etool
+
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame import *
+
+
+class TDTestCase(TBase):
+        
+
+    def insertData(self):
+        tdLog.info(f"insert data.")
+        # taosBenchmark run
+        jfile = etool.curFile(__file__, "snapshot.json")
+        etool.runBenchmark(json=jfile)
+
+        tdSql.execute(f"use {self.db}")
+        # set insert data information
+        self.childtable_count = 10
+        self.insert_rows      = 100000
+        self.timestamp_step   = 10000
+
+    def doAction(self):
+        tdLog.info(f"do action.")
+        self.flushDb()
+
+        # split vgroups
+        self.splitVGroups()
+        self.trimDb()
+        self.checkAggCorrect()
+
+        # balance vgroups
+        self.balanceVGroupLeader()
+
+        # replica to 1
+        self.alterReplica(1)
+        self.checkAggCorrect()
+        self.compactDb()
+        self.alterReplica(3)
+
+        vgids = self.getVGroup(self.db)
+        selid = random.choice(vgids)
+        self.balanceVGroupLeaderOn(selid)
+
+        
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # insert data
+        self.insertData()
+
+        # check insert data correct
+        self.checkInsertCorrect()
+
+        # save
+        self.snapshotAgg()
+
+        # do action
+        self.doAction()
+
+        # check save agg result correct
+        self.checkAggCorrect()
+
+        # check insert correct again
+        self.checkInsertCorrect()
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/army/frame/caseBase.py

@@ -15,6 +15,7 @@ import sys
 import os
 import time
 import datetime
+import random
 
 from frame.log import *
 from frame.sql import *
@@ -42,17 +43,13 @@ class TBase:
         self.db     = "db"
         self.stb    = "stb"
 
-        # variant in taosBenchmark json
-        self.childtable_count = 2
-        self.insert_rows = 1000000
-        self.timestamp_step = 1000
-
         # sql 
         self.sqlSum = f"select sum(ic) from {self.stb}"
         self.sqlMax = f"select max(ic) from {self.stb}"
         self.sqlMin = f"select min(ic) from {self.stb}"
         self.sqlAvg = f"select avg(ic) from {self.stb}"
-
+        self.sqlFirst = f"select first(ts) from {self.stb}"
+        self.sqlLast  = f"select last(ts) from {self.stb}"
 
     # stop
     def stop(self):
@@ -75,6 +72,50 @@ class TBase:
     def dropDb(self, show = False):
         tdSql.execute(f"drop database {self.db}", show = show)
 
+    def splitVGroups(self):
+        vgids = self.getVGroup(self.db)
+        selid = random.choice(vgids)
+        sql = f"split vgroup {selid}"
+        tdSql.execute(sql, show=True)
+        if self.waitTransactionZero() is False:
+            tdLog.exit(f"{sql} transaction not finished")
+            return False
+        return True
+    
+
+    def alterReplica(self, replica):
+        sql = f"alter database {self.db} replica {replica}"
+        tdSql.execute(sql, show=True)
+        if self.waitTransactionZero() is False:
+            tdLog.exit(f"{sql} transaction not finished")
+            return False
+        return True
+
+    def balanceVGroup(self):
+        sql = f"balance vgroup"
+        tdSql.execute(sql, show=True)
+        if self.waitTransactionZero() is False:
+            tdLog.exit(f"{sql} transaction not finished")
+            return False
+        return True
+    
+    def balanceVGroupLeader(self):
+        sql = f"balance vgroup leader"
+        tdSql.execute(sql, show=True)
+        if self.waitTransactionZero() is False:
+            tdLog.exit(f"{sql} transaction not finished")
+            return False
+        return True
+
+
+    def balanceVGroupLeaderOn(self, vgId):
+        sql = f"balance vgroup leader on {vgId}"
+        tdSql.execute(sql, show=True)
+        if self.waitTransactionZero() is False:
+            tdLog.exit(f"{sql} transaction not finished")
+            return False
+        return True
+
 
 #
 #  check db correct
@@ -95,12 +136,13 @@ class TBase:
         tdSql.checkAgg(sql, 0)
 
     # save agg result
-    def snapshotAgg(self):
+    def snapshotAgg(self):        
-        
         self.sum =  tdSql.getFirstValue(self.sqlSum)
         self.avg =  tdSql.getFirstValue(self.sqlAvg)
         self.min =  tdSql.getFirstValue(self.sqlMin)
         self.max =  tdSql.getFirstValue(self.sqlMax)
+        self.first = tdSql.getFirstValue(self.sqlFirst)
+        self.last  = tdSql.getFirstValue(self.sqlLast)
 
     # check agg 
     def checkAggCorrect(self):
@@ -108,3 +150,41 @@ class TBase:
         tdSql.checkAgg(self.sqlAvg, self.avg)
         tdSql.checkAgg(self.sqlMin, self.min)
         tdSql.checkAgg(self.sqlMax, self.max)
+        tdSql.checkAgg(self.sqlFirst, self.first)
+        tdSql.checkAgg(self.sqlLast,  self.last)
+
+
+#
+#   get db information
+#
+
+    # get vgroups
+    def getVGroup(self, db_name):
+        vgidList = []
+        sql = f"select vgroup_id from information_schema.ins_vgroups where db_name='{db_name}'"
+        res = tdSql.getResult(sql)
+        rows = len(res)
+        for i in range(rows):
+            vgidList.append(res[i][0])
+
+        return vgidList
+    
+
+
+#
+#   util 
+#
+    
+    # wait transactions count to zero , return False is translation not finished
+    def waitTransactionZero(self, seconds = 300, interval = 1):
+        # wait end
+        for i in range(seconds):
+            sql ="show transactions;"
+            rows = tdSql.query(sql)
+            if rows == 0:
+                tdLog.info("transaction count became zero.")
+                return True
+            #tdLog.info(f"i={i} wait ...")
+            time.sleep(interval)
+        
+        return False    

tests/parallel_test/cases.task

@@ -10,7 +10,7 @@
 #
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1
-
+,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
 
 
 #

tools/shell/src/shellAuto.c

@@ -84,8 +84,8 @@ SWords shellCommands[] = {
     {"alter topic", 0, 0, NULL},
     {"alter user <user_name> <user_actions> <anyword> ;", 0, 0, NULL},
 #ifdef TD_ENTERPRISE
-    {"balance vgroup;", 0, 0, NULL},
+    {"balance vgroup ;", 0, 0, NULL},
-    {"balance vgroup leader <vgroup_id>", 0, 0, NULL},
+    {"balance vgroup leader on <vgroup_id>", 0, 0, NULL},
 #endif
 
     // 20
@@ -531,8 +531,8 @@ void showHelp() {
   printf(
       "\n\n\
   ----- special commands on enterpise version ----- \n\
-    balance vgroup; \n\
+    balance vgroup ;\n\
-    balance vgroup leader <vgroup_id> \n\
+    balance vgroup leader on <vgroup_id> \n\
     compact database <db_name>; \n\
     redistribute vgroup <vgroup_id> dnode <dnode_id> ;\n\
     split vgroup <vgroup_id>;");