Merge pull request #26056 from taosdata/fix/TD-30331-3.0

fix: s3 with different bucket name 1~20

tests/army/cluster/incSnapshot.py

@@ -0,0 +1,110 @@
+import taos
+import sys
+import os
+import subprocess
+import glob
+import shutil
+import time
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.srvCtl import *
+from frame.caseBase import *
+from frame import *
+from frame.autogen import *
+# from frame.server.dnodes import *
+# from frame.server.cluster import *
+
+
+class TDTestCase(TBase):
+    updatecfgDict = {
+        'slowLogScope':"query"
+    }
+
+    def init(self, conn, logSql, replicaVar=3):
+        super(TDTestCase, self).init(conn, logSql, replicaVar=3, db="snapshot", checkColName="c1")
+        self.valgrind = 0
+        self.childtable_count = 10
+        # tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def run(self):
+        tdSql.prepare()
+        autoGen = AutoGen()
+        autoGen.create_db(self.db, 2, 3)
+        tdSql.execute(f"use {self.db}")
+        autoGen.create_stable(self.stb, 5, 10, 8, 8)
+        autoGen.create_child(self.stb, "d", self.childtable_count)
+        autoGen.insert_data(1000)
+        tdSql.execute(f"flush database {self.db}")
+        sc.dnodeStop(3)
+        # clusterDnodes.stoptaosd(1)
+        # clusterDnodes.starttaosd(3)
+        # time.sleep(5)
+        # clusterDnodes.stoptaosd(2)
+        # clusterDnodes.starttaosd(1)
+        # time.sleep(5)
+        autoGen.insert_data(5000, True)
+        self.flushDb(True)
+        # wait flush operation over
+        time.sleep(5)
+
+        # sql = 'show vnodes;'
+        # while True:
+        #     bFinish = True
+        #     param_list = tdSql.query(sql, row_tag=True)
+        #     for param in param_list:
+        #         if param[3] == 'leading' or param[3] == 'following':
+        #             bFinish = False
+        #             break
+        #     if bFinish:
+        #         break
+        self.snapshotAgg()
+        time.sleep(10)
+        sc.dnodeStopAll()
+        for i in range(1, 4):
+            path = clusterDnodes.getDnodeDir(i)
+            dnodesRootDir = os.path.join(path,"data","vnode", "vnode*")
+            dirs = glob.glob(dnodesRootDir)
+            for dir in dirs:
+                if os.path.isdir(dir):
+                    self.remove_directory(os.path.join(dir, "wal"))
+
+        sc.dnodeStart(1)
+        sc.dnodeStart(2)
+        sc.dnodeStart(3)
+        sql = "show vnodes;"
+        time.sleep(10)
+        while True:
+            bFinish = True
+            param_list = tdSql.query(sql, row_tag=True)
+            for param in param_list:
+                if param[3] == 'offline':
+                    tdLog.exit(
+                        "dnode synchronous fail dnode id: %d, vgroup id:%d status offline" % (param[0], param[1]))
+                if param[3] == 'leading' or param[3] == 'following':
+                    bFinish = False
+                    break
+            if bFinish:
+                break
+
+        self.timestamp_step = 1000
+        self.insert_rows = 6000
+        self.checkInsertCorrect()
+        self.checkAggCorrect()
+
+    def remove_directory(self, directory):
+        try:
+            shutil.rmtree(directory)
+            tdLog.debug("delete dir: %s " % (directory))
+        except OSError as e:
+            tdLog.exit("delete fail dir: %s " % (directory))
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/army/frame/etime.py

@@ -19,5 +19,3 @@ import sys
 import os
 import time
 import datetime
-
-

tests/army/frame/eutil.py

@@ -0,0 +1,28 @@
+###################################################################
+#           Copyright (c) 2023 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+#
+# about tools funciton extension
+#
+
+import sys
+import os
+import time
+import datetime
+import psutil
+
+
+# cpu frequent as random
+def cpuRand(max):
+    decimal = int(str(psutil.cpu_freq().current).split(".")[1])
+    return decimal % max

tests/army/s3/s3Basic.py

@@ -19,6 +19,7 @@ import taos
 import frame
 import frame.etool
 import frame.eos
+import frame.eutil
 
 from frame.log import *
 from frame.cases import *
@@ -46,17 +47,20 @@ for test:
 
 
 class TDTestCase(TBase):
+    index = eutil.cpuRand(20) + 1
+    bucketName = f"ci-bucket{index}"
     updatecfgDict = {
         "supportVnodes":"1000",
         's3EndPoint': 'http://192.168.1.52:9000', 
         's3AccessKey': 'zOgllR6bSnw2Ah3mCNel:cdO7oXAu3Cqdb1rUdevFgJMi0LtRwCXdWKQx4bhX', 
-        's3BucketName': 'ci-bucket',
+        's3BucketName': f'{bucketName}',
         's3PageCacheSize': '10240',
         "s3UploadDelaySec": "10",
         's3MigrateIntervalSec': '600',
         's3MigrateEnabled': '1'
     }
 
+    tdLog.info(f"assign bucketName is {bucketName}\n")
     maxFileSize = (128 + 10) * 1014 * 1024 # add 10M buffer
 
     def insertData(self):
@@ -241,10 +245,9 @@ class TDTestCase(TBase):
 
     #
     def preDb(self, vgroups):
-
-        cnt = int(time.time())%3 + 1
+        cnt = int(time.time())%2 + 1
         for i in range(cnt):
-             vg = int(time.time()*1000)%10 + 1
+             vg = eutil.cpuRand(9) + 1
              sql = f"create database predb vgroups {vg}"
              tdSql.execute(sql, show=True)
              sql = "drop database predb"

tests/parallel_test/cases.task

@@ -10,26 +10,26 @@
 #
 # army-test
 #
-,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
-,,y,army,./pytest.sh python3 ./test.py -f enterprise/db-encrypt/basic.py
-,,n,army,python3 ./test.py -f enterprise/s3/s3Basic.py -N 3
-,,y,army,./pytest.sh python3 ./test.py -f community/cluster/snapshot.py -N 3 -L 3 -D 2
-,,y,army,./pytest.sh python3 ./test.py -f community/query/function/test_func_elapsed.py
-,,y,army,./pytest.sh python3 ./test.py -f community/query/test_join.py
-,,y,army,./pytest.sh python3 ./test.py -f community/query/test_compare.py
-,,y,army,./pytest.sh python3 ./test.py -f community/insert/test_column_tag_boundary.py
-,,y,army,./pytest.sh python3 ./test.py -f community/query/fill/fill_desc.py -N 3 -L 3 -D 2
-,,y,army,./pytest.sh python3 ./test.py -f community/cluster/incSnapshot.py -N 3
-,,y,army,./pytest.sh python3 ./test.py -f community/query/query_basic.py -N 3
-,,y,army,./pytest.sh python3 ./test.py -f community/query/accuracy/test_query_accuracy.py
-,,y,army,./pytest.sh python3 ./test.py -f community/insert/insert_basic.py -N 3
-,,y,army,./pytest.sh python3 ./test.py -f community/cluster/splitVgroupByLearner.py -N 3
-,,n,army,python3 ./test.py -f community/cmdline/fullopt.py
-,,n,army,python3 ./test.py -f community/query/show.py -N 3
-,,n,army,python3 ./test.py -f enterprise/alter/alterConfig.py -N 3
-,,y,army,./pytest.sh python3 ./test.py -f community/query/subquery/subqueryBugs.py -N 3
-,,y,army,./pytest.sh python3 ./test.py -f community/storage/oneStageComp.py -N 3 -L 3 -D 1
-,,y,army,./pytest.sh python3 ./test.py -f community/storage/compressBasic.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f multi-level/mlevel_basic.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f db-encrypt/basic.py
+,,n,army,python3 ./test.py -f s3/s3Basic.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f cluster/snapshot.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f query/function/test_func_elapsed.py
+,,y,army,./pytest.sh python3 ./test.py -f query/test_join.py
+,,y,army,./pytest.sh python3 ./test.py -f query/test_compare.py
+,,y,army,./pytest.sh python3 ./test.py -f insert/test_column_tag_boundary.py
+,,y,army,./pytest.sh python3 ./test.py -f query/fill/fill_desc.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f cluster/incSnapshot.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f query/query_basic.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f query/accuracy/test_query_accuracy.py
+,,y,army,./pytest.sh python3 ./test.py -f insert/insert_basic.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f cluster/splitVgroupByLearner.py -N 3
+,,n,army,python3 ./test.py -f cmdline/fullopt.py
+,,n,army,python3 ./test.py -f query/show.py -N 3
+,,n,army,python3 ./test.py -f alter/alterConfig.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f query/subquery/subqueryBugs.py -N 3
+,,y,army,./pytest.sh python3 ./test.py -f storage/oneStageComp.py -N 3 -L 3 -D 1
+,,y,army,./pytest.sh python3 ./test.py -f storage/compressBasic.py -N 3
 
 #
 # system test