Merge pull request #24290 from taosdata/case/TD-18808-3.0

s3 with minio add test case  server on 192.168.1.52

source/common/src/tglobal.c

@@ -789,7 +789,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
                   CFG_DYN_ENT_SERVER) != 0)
     return -1;
-  if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
+  if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
                   CFG_DYN_ENT_SERVER) != 0)
     return -1;
 

tests/army/enterprise/s3/s3_basic.json

@@ -0,0 +1,58 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "connection_pool_size": 8,
+    "num_of_records_per_req": 2000,
+    "thread_count": 2,
+    "create_table_thread_count": 1,
+    "confirm_parameter_prompt": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "db",
+                "drop": "yes",
+                "vgroups": 2,
+                "replica": 1,
+                "duration":"1d",
+                "keep": "3d,6d,30d"
+            },
+            "super_tables": [
+                {
+                    "name": "stb",
+                    "child_table_exists": "no",
+                    "childtable_count": 4,
+                    "insert_rows": 1000000,
+                    "childtable_prefix": "d",
+                    "insert_mode": "taosc",
+                    "timestamp_step": 1000,
+                    "start_timestamp":"now-13d",
+                    "columns": [
+                        { "type": "bool",        "name": "bc"},
+                        { "type": "float",       "name": "fc" },
+                        { "type": "double",      "name": "dc"},
+                        { "type": "tinyint",     "name": "ti", "values":["1"]},
+                        { "type": "smallint",    "name": "si" },
+                        { "type": "int",         "name": "ic" },
+                        { "type": "bigint",      "name": "bi" },
+                        { "type": "utinyint",    "name": "uti"},
+                        { "type": "usmallint",   "name": "usi"},
+                        { "type": "uint",        "name": "ui" },
+                        { "type": "ubigint",     "name": "ubi"},
+                        { "type": "binary",      "name": "bin", "len": 32},
+                        { "type": "nchar",       "name": "nch", "len": 64}
+                    ],
+                    "tags": [
+                        {"type": "tinyint", "name": "groupid","max": 10,"min": 1},
+                        {"name": "location","type": "binary", "len": 16, "values":
+                           ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}

tests/army/enterprise/s3/s3_basic.py

@@ -0,0 +1,114 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+
+import taos
+import frame
+import frame.etool
+import frame.eos
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame.srvCtl import *
+from frame import *
+from frame.eos import *
+
+#  
+# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+#
+
+'''
+s3EndPoint     http://192.168.1.52:9000
+s3AccessKey    MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+s3BucketName   ci-bucket
+s3UploadDelaySec 60
+'''
+
+
+class TDTestCase(TBase):
+    updatecfgDict = {
+        's3EndPoint': 'http://192.168.1.52:9000', 
+        's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', 
+        's3BucketName': 'ci-bucket',
+        's3BlockSize': '10240',
+        's3BlockCacheSize': '320',
+        's3PageCacheSize': '10240',
+        's3UploadDelaySec':'60'
+    }    
+
+    def insertData(self):
+        tdLog.info(f"insert data.")
+        # taosBenchmark run
+        json = etool.curFile(__file__, "s3_basic.json")
+        etool.runBenchmark(json=json)
+
+        tdSql.execute(f"use {self.db}")
+        # set insert data information
+        self.childtable_count = 4
+        self.insert_rows = 1000000
+        self.timestamp_step = 1000
+
+    def doAction(self):
+        tdLog.info(f"do action.")
+        self.flushDb()
+        self.compactDb()
+
+        # sleep 70s
+        tdLog.info(f"wait 65s ...")
+        time.sleep(65)
+        self.trimDb(True)
+
+        rootPath = sc.clusterRootPath()
+        cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data"
+        tdLog.info(cmd)
+        loop = 0
+        while len(eos.runRetList(cmd)) > 0 and loop < 40:
+            time.sleep(5)
+            self.trimDb(True)
+            loop += 1
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # insert data
+        self.insertData()
+
+        # check insert data correct
+        self.checkInsertCorrect()
+
+        # save
+        self.snapshotAgg()
+
+        # do action
+        self.doAction()
+
+        # check save agg result correct
+        self.checkAggCorrect()
+
+        # check insert correct again
+        self.checkInsertCorrect()
+
+        # drop database and free s3 file
+        self.dropDb()
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/army/frame/caseBase.py

@@ -63,14 +63,18 @@ class TBase:
 #   db action
 #         
 
-    def trimDb(self):
+    def trimDb(self, show = False):
-        tdSql.execute(f"trim database {self.db}")
+        tdSql.execute(f"trim database {self.db}", show = show)
 
-    def compactDb(self):
+    def compactDb(self, show = False):
-        tdSql.execute(f"compact database {self.db}")
+        tdSql.execute(f"compact database {self.db}", show = show)
+
+    def flushDb(self, show = False):
+        tdSql.execute(f"flush database {self.db}", show = show)
+
+    def dropDb(self, show = False):
+        tdSql.execute(f"drop database {self.db}", show = show)
 
-    def flushDb(self):
-        tdSql.execute(f"flush database {self.db}")
 
 #
 #  check db correct

tests/army/frame/eos.py

@@ -20,11 +20,16 @@ import os
 import time
 import datetime
 import platform
+import subprocess
 
 # if windows platform return True
 def isWin():
     return platform.system().lower() == 'windows'
 
+#
+#  execute programe
+#
+
 # wait util execute file finished 
 def exe(file):
     return os.system(file)
@@ -34,3 +39,19 @@ def exeNoWait(file):
     print("exe no wait")
 
 
+# run return output and error
+def run(command):
+    process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    process.wait(3)
+
+    output = process.stdout.read().decode(encoding="gbk")
+    error = process.stderr.read().decode(encoding="gbk")
+
+    return output, error
+
+
+# return list after run
+def runRetList(command):
+    lines = []
+    output,error = run(command)
+    return output.splitlines()

tests/army/frame/epath.py

@@ -51,3 +51,6 @@ def binPath():
 
 def binFile(filename):
     return binPath() + filename
+
+
+

tests/army/frame/srvCtl.py

@@ -16,6 +16,8 @@ import os
 import time
 import datetime
 
+from frame.server.dnodes import *
+
 class srvCtl:
     def __init__(self):
         # record server information
@@ -24,4 +26,19 @@ class srvCtl:
         self.mLevel = 0
         self.mLevelDisk = 0
 
+    #
+    #  about path
+    #
+
+    # get cluster root path like /root/TDinternal/sim/ 
+    def clusterRootPath(self):
+        return tdDnodes.getDnodesRootDir()
+
+    # return dnode data files list
+    def dnodeDataFiles(self, idx):
+        files = []
+        return files
+
+    
+
 sc = srvCtl()

tests/parallel_test/cases.task

@@ -5,10 +5,17 @@
 #unit-test
 ,,y,unit-test,bash test.sh
 
-#army-test
+#
+# army-test
+#
 ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
+,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1
 
-#system test
+
+
+#
+# system test
+#
 ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py