From 1e8244f3c692cf181fa1a29d1c4d4b3a7493fa52 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 10:31:15 +0800 Subject: [PATCH 1/4] feat: add s3 minio ci case --- source/common/src/tglobal.c | 2 +- tests/army/enterprise/s3/s3_basic.json | 58 ++++++++++++++ tests/army/enterprise/s3/s3_basic.py | 105 +++++++++++++++++++++++++ tests/army/frame/caseBase.py | 4 + tests/army/frame/srvCtl.py | 5 ++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 tests/army/enterprise/s3/s3_basic.json create mode 100644 tests/army/enterprise/s3/s3_basic.py diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fdac3882c3..d6d1e1810d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -789,7 +789,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json new file mode 100644 index 0000000000..1c2b9274d2 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"1d", + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 4, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py new file mode 100644 index 0000000000..12c0720112 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.py @@ -0,0 +1,105 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * + +# +# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +# + +''' +s3EndPoint http://192.168.1.52:9000 +s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +s3BucketName ci-bucket +s3UploadDelaySec 60 +''' + + +class TDTestCase(TBase): + updatecfgDict = { + 's3EndPoint': 'http://192.168.1.52:9000', + 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', + 's3BucketName': 'ci-bucket', + 's3UploadDelaySec':'60' + } + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = etool.curFile(__file__, "mlevel_basic.json") + etool.runBenchmark(json=json) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 4 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + def doAction(self): + tdLog.info(f"do action.") + self.flushDb() + self.compactDb() + + # sleep 70s + time.sleep(70) + self.trimDb() + + loop = 0 + while len(sc.dnodeDataFiles()) > 0 and loop < 10: + time.sleep(10) + self.trimDb() + loop += 1 + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + # drop database and free s3 file + self.dropDb() + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 441196f050..6009111d43 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -72,6 +72,10 @@ class TBase: def flushDb(self): tdSql.execute(f"flush database {self.db}") + def dropDb(self): + tdSql.execute(f"drop database {self.db}") + + # # check db correct # diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index e7ef08cde9..99f503d4b6 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -24,4 +24,9 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # return dnode data files list + def dnodeDataFiles(self, idx): + files = [] + return files + sc = srvCtl() \ No newline at end of file From 64164f5684289af8a36ec52a2d41845359536551 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:39:39 +0800 Subject: [PATCH 2/4] fix: run s3_basic.py ok --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 16 ++++++++++++---- tests/army/frame/caseBase.py | 16 ++++++++-------- tests/army/frame/eos.py | 21 +++++++++++++++++++++ tests/army/frame/epath.py | 3 +++ tests/army/frame/srvCtl.py | 12 ++++++++++++ 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 1c2b9274d2..169da290bf 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-12d", + "start_timestamp":"now-15d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 12c0720112..84c91209b3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -17,6 +17,7 @@ import time import taos import frame import frame.etool +import frame.eos from frame.log import * from frame.cases import * @@ -24,6 +25,7 @@ from frame.sql import * from frame.caseBase import * from frame.srvCtl import * from frame import * +from frame.eos import * # # 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa @@ -42,13 +44,16 @@ class TDTestCase(TBase): 's3EndPoint': 'http://192.168.1.52:9000', 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', 's3BucketName': 'ci-bucket', + 's3BlockSize': '10240', + 's3BlockCacheSize': '320', + 's3PageCacheSize': '10240', 's3UploadDelaySec':'60' } def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run - json = etool.curFile(__file__, "mlevel_basic.json") + json = etool.curFile(__file__, "s3_basic.json") etool.runBenchmark(json=json) tdSql.execute(f"use {self.db}") @@ -63,12 +68,15 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s + tdLog.info(f"wait 70s ...") time.sleep(70) self.trimDb() - + + rootPath = sc.clusterRootPath() + cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" loop = 0 - while len(sc.dnodeDataFiles()) > 0 and loop < 10: - time.sleep(10) + while len(eos.runRetList(cmd)) > 0 and loop < 40: + time.sleep(5) self.trimDb() loop += 1 diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 6009111d43..ad6e744fe6 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -63,17 +63,17 @@ class TBase: # db action # - def trimDb(self): - tdSql.execute(f"trim database {self.db}") + def trimDb(self, show = False): + tdSql.execute(f"trim database {self.db}", show = show) - def compactDb(self): - tdSql.execute(f"compact database {self.db}") + def compactDb(self, show = False): + tdSql.execute(f"compact database {self.db}", show = show) - def flushDb(self): - tdSql.execute(f"flush database {self.db}") + def flushDb(self, show = False): + tdSql.execute(f"flush database {self.db}", show = show) - def dropDb(self): - tdSql.execute(f"drop database {self.db}") + def dropDb(self, show = False): + tdSql.execute(f"drop database {self.db}", show = show) # diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index dcb63ff3aa..b24be74393 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -20,11 +20,16 @@ import os import time import datetime import platform +import subprocess # if windows platform return True def isWin(): return platform.system().lower() == 'windows' +# +# execute programe +# + # wait util execute file finished def exe(file): return os.system(file) @@ -34,3 +39,19 @@ def exeNoWait(file): print("exe no wait") +# run return output and error +def run(command): + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait(3) + + output = process.stdout.read().decode(encoding="gbk") + error = process.stderr.read().decode(encoding="gbk") + + return output, error + + +# return list after run +def runRetList(command): + lines = [] + output,error = run(command) + return output.splitlines() diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py index edff9c78a4..c7796befbc 100644 --- a/tests/army/frame/epath.py +++ b/tests/army/frame/epath.py @@ -51,3 +51,6 @@ def binPath(): def binFile(filename): return binPath() + filename + + + diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 99f503d4b6..75f0d79513 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -16,6 +16,8 @@ import os import time import datetime +from frame.server.dnodes import * + class srvCtl: def __init__(self): # record server information @@ -24,9 +26,19 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # + # about path + # + + # get cluster root path like /root/TDinternal/sim/ + def clusterRootPath(self): + return tdDnodes.path + # return dnode data files list def dnodeDataFiles(self, idx): files = [] return files + + sc = srvCtl() \ No newline at end of file From 26023864f14ab5a6b6926bca68a6a43c2cb12a8a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:48:36 +0800 Subject: [PATCH 3/4] fix: add show path information --- tests/army/enterprise/s3/s3_basic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 84c91209b3..85beb195c3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -70,14 +70,15 @@ class TDTestCase(TBase): # sleep 70s tdLog.info(f"wait 70s ...") time.sleep(70) - self.trimDb() + self.trimDb(True) rootPath = sc.clusterRootPath() cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: time.sleep(5) - self.trimDb() + self.trimDb(True) loop += 1 # run From 1600f0654d6c3f46aae1242199aa61f7af223a2a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 15:16:29 +0800 Subject: [PATCH 4/4] fix: s3 for minio finished --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 6 +++--- tests/army/frame/srvCtl.py | 2 +- tests/parallel_test/cases.task | 11 +++++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 169da290bf..e56a18e757 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-15d", + "start_timestamp":"now-13d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 85beb195c3..389b8a52c7 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -68,12 +68,12 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s - tdLog.info(f"wait 70s ...") - time.sleep(70) + tdLog.info(f"wait 65s ...") + time.sleep(65) self.trimDb(True) rootPath = sc.clusterRootPath() - cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 75f0d79513..155c83094d 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -32,7 +32,7 @@ class srvCtl: # get cluster root path like /root/TDinternal/sim/ def clusterRootPath(self): - return tdDnodes.path + return tdDnodes.getDnodesRootDir() # return dnode data files list def dnodeDataFiles(self, idx): diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b5cde02d25..e7395a26f1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -5,10 +5,17 @@ #unit-test ,,y,unit-test,bash test.sh -#army-test +# +# army-test +# ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 -#system test + + +# +# system test +# ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py