feat: add s3 minio ci case

source/common/src/tglobal.c

@@ -789,7 +789,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
                   CFG_DYN_ENT_SERVER) != 0)
     return -1;
-  if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
+  if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
                   CFG_DYN_ENT_SERVER) != 0)
     return -1;
 

tests/army/enterprise/s3/s3_basic.json

@@ -0,0 +1,58 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "connection_pool_size": 8,
+    "num_of_records_per_req": 2000,
+    "thread_count": 2,
+    "create_table_thread_count": 1,
+    "confirm_parameter_prompt": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "db",
+                "drop": "yes",
+                "vgroups": 2,
+                "replica": 1,
+                "duration":"1d",
+                "keep": "3d,6d,30d"
+            },
+            "super_tables": [
+                {
+                    "name": "stb",
+                    "child_table_exists": "no",
+                    "childtable_count": 4,
+                    "insert_rows": 1000000,
+                    "childtable_prefix": "d",
+                    "insert_mode": "taosc",
+                    "timestamp_step": 1000,
+                    "start_timestamp":"now-12d",
+                    "columns": [
+                        { "type": "bool",        "name": "bc"},
+                        { "type": "float",       "name": "fc" },
+                        { "type": "double",      "name": "dc"},
+                        { "type": "tinyint",     "name": "ti", "values":["1"]},
+                        { "type": "smallint",    "name": "si" },
+                        { "type": "int",         "name": "ic" },
+                        { "type": "bigint",      "name": "bi" },
+                        { "type": "utinyint",    "name": "uti"},
+                        { "type": "usmallint",   "name": "usi"},
+                        { "type": "uint",        "name": "ui" },
+                        { "type": "ubigint",     "name": "ubi"},
+                        { "type": "binary",      "name": "bin", "len": 32},
+                        { "type": "nchar",       "name": "nch", "len": 64}
+                    ],
+                    "tags": [
+                        {"type": "tinyint", "name": "groupid","max": 10,"min": 1},
+                        {"name": "location","type": "binary", "len": 16, "values":
+                           ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}

tests/army/enterprise/s3/s3_basic.py

@@ -0,0 +1,105 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+
+import taos
+import frame
+import frame.etool
+
+from frame.log import *
+from frame.cases import *
+from frame.sql import *
+from frame.caseBase import *
+from frame.srvCtl import *
+from frame import *
+
+#  
+# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+#
+
+'''
+s3EndPoint     http://192.168.1.52:9000
+s3AccessKey    MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
+s3BucketName   ci-bucket
+s3UploadDelaySec 60
+'''
+
+
+class TDTestCase(TBase):
+    updatecfgDict = {
+        's3EndPoint': 'http://192.168.1.52:9000', 
+        's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', 
+        's3BucketName': 'ci-bucket',
+        's3UploadDelaySec':'60'
+    }    
+
+    def insertData(self):
+        tdLog.info(f"insert data.")
+        # taosBenchmark run
+        json = etool.curFile(__file__, "mlevel_basic.json")
+        etool.runBenchmark(json=json)
+
+        tdSql.execute(f"use {self.db}")
+        # set insert data information
+        self.childtable_count = 4
+        self.insert_rows = 1000000
+        self.timestamp_step = 1000
+
+    def doAction(self):
+        tdLog.info(f"do action.")
+        self.flushDb()
+        self.compactDb()
+
+        # sleep 70s
+        time.sleep(70)
+        self.trimDb()
+        
+        loop = 0
+        while len(sc.dnodeDataFiles()) > 0 and loop < 10:
+            time.sleep(10)
+            self.trimDb()
+            loop += 1
+
+    # run
+    def run(self):
+        tdLog.debug(f"start to excute {__file__}")
+
+        # insert data
+        self.insertData()
+
+        # check insert data correct
+        self.checkInsertCorrect()
+
+        # save
+        self.snapshotAgg()
+
+        # do action
+        self.doAction()
+
+        # check save agg result correct
+        self.checkAggCorrect()
+
+        # check insert correct again
+        self.checkInsertCorrect()
+
+        # drop database and free s3 file
+        self.dropDb()
+
+        tdLog.success(f"{__file__} successfully executed")
+
+        
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

tests/army/frame/caseBase.py

@@ -72,6 +72,10 @@ class TBase:
     def flushDb(self):
         tdSql.execute(f"flush database {self.db}")
 
+    def dropDb(self):
+        tdSql.execute(f"drop database {self.db}")
+
+
 #
 #  check db correct
 #                

tests/army/frame/srvCtl.py

@@ -24,4 +24,9 @@ class srvCtl:
         self.mLevel = 0
         self.mLevelDisk = 0
 
+    # return dnode data files list
+    def dnodeDataFiles(self, idx):
+        files = []
+        return files
+
 sc = srvCtl()