Merge pull request #24290 from taosdata/case/TD-18808-3.0
s3 with minio add test case server on 192.168.1.52
This commit is contained in:
Alex Duan
GitHub
commit
1debd50a57
|
|
@ -789,7 +789,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
|
|||
if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_ENT_SERVER) != 0)
|
||||
return -1;
|
||||
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
|
||||
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
|
||||
CFG_DYN_ENT_SERVER) != 0)
|
||||
return -1;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"connection_pool_size": 8,
|
||||
"num_of_records_per_req": 2000,
|
||||
"thread_count": 2,
|
||||
"create_table_thread_count": 1,
|
||||
"confirm_parameter_prompt": "no",
|
||||
"databases": [
|
||||
{
|
||||
"dbinfo": {
|
||||
"name": "db",
|
||||
"drop": "yes",
|
||||
"vgroups": 2,
|
||||
"replica": 1,
|
||||
"duration":"1d",
|
||||
"keep": "3d,6d,30d"
|
||||
},
|
||||
"super_tables": [
|
||||
{
|
||||
"name": "stb",
|
||||
"child_table_exists": "no",
|
||||
"childtable_count": 4,
|
||||
"insert_rows": 1000000,
|
||||
"childtable_prefix": "d",
|
||||
"insert_mode": "taosc",
|
||||
"timestamp_step": 1000,
|
||||
"start_timestamp":"now-13d",
|
||||
"columns": [
|
||||
{ "type": "bool", "name": "bc"},
|
||||
{ "type": "float", "name": "fc" },
|
||||
{ "type": "double", "name": "dc"},
|
||||
{ "type": "tinyint", "name": "ti", "values":["1"]},
|
||||
{ "type": "smallint", "name": "si" },
|
||||
{ "type": "int", "name": "ic" },
|
||||
{ "type": "bigint", "name": "bi" },
|
||||
{ "type": "utinyint", "name": "uti"},
|
||||
{ "type": "usmallint", "name": "usi"},
|
||||
{ "type": "uint", "name": "ui" },
|
||||
{ "type": "ubigint", "name": "ubi"},
|
||||
{ "type": "binary", "name": "bin", "len": 32},
|
||||
{ "type": "nchar", "name": "nch", "len": 64}
|
||||
],
|
||||
"tags": [
|
||||
{"type": "tinyint", "name": "groupid","max": 10,"min": 1},
|
||||
{"name": "location","type": "binary", "len": 16, "values":
|
||||
["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
import taos
|
||||
import frame
|
||||
import frame.etool
|
||||
import frame.eos
|
||||
|
||||
from frame.log import *
|
||||
from frame.cases import *
|
||||
from frame.sql import *
|
||||
from frame.caseBase import *
|
||||
from frame.srvCtl import *
|
||||
from frame import *
|
||||
from frame.eos import *
|
||||
|
||||
#
|
||||
# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
|
||||
#
|
||||
|
||||
'''
|
||||
s3EndPoint http://192.168.1.52:9000
|
||||
s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa
|
||||
s3BucketName ci-bucket
|
||||
s3UploadDelaySec 60
|
||||
'''
|
||||
|
||||
|
||||
class TDTestCase(TBase):
|
||||
updatecfgDict = {
|
||||
's3EndPoint': 'http://192.168.1.52:9000',
|
||||
's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa',
|
||||
's3BucketName': 'ci-bucket',
|
||||
's3BlockSize': '10240',
|
||||
's3BlockCacheSize': '320',
|
||||
's3PageCacheSize': '10240',
|
||||
's3UploadDelaySec':'60'
|
||||
}
|
||||
|
||||
def insertData(self):
|
||||
tdLog.info(f"insert data.")
|
||||
# taosBenchmark run
|
||||
json = etool.curFile(__file__, "s3_basic.json")
|
||||
etool.runBenchmark(json=json)
|
||||
|
||||
tdSql.execute(f"use {self.db}")
|
||||
# set insert data information
|
||||
self.childtable_count = 4
|
||||
self.insert_rows = 1000000
|
||||
self.timestamp_step = 1000
|
||||
|
||||
def doAction(self):
|
||||
tdLog.info(f"do action.")
|
||||
self.flushDb()
|
||||
self.compactDb()
|
||||
|
||||
# sleep 70s
|
||||
tdLog.info(f"wait 65s ...")
|
||||
time.sleep(65)
|
||||
self.trimDb(True)
|
||||
|
||||
rootPath = sc.clusterRootPath()
|
||||
cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data"
|
||||
tdLog.info(cmd)
|
||||
loop = 0
|
||||
while len(eos.runRetList(cmd)) > 0 and loop < 40:
|
||||
time.sleep(5)
|
||||
self.trimDb(True)
|
||||
loop += 1
|
||||
|
||||
# run
|
||||
def run(self):
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
|
||||
# insert data
|
||||
self.insertData()
|
||||
|
||||
# check insert data correct
|
||||
self.checkInsertCorrect()
|
||||
|
||||
# save
|
||||
self.snapshotAgg()
|
||||
|
||||
# do action
|
||||
self.doAction()
|
||||
|
||||
# check save agg result correct
|
||||
self.checkAggCorrect()
|
||||
|
||||
# check insert correct again
|
||||
self.checkInsertCorrect()
|
||||
|
||||
# drop database and free s3 file
|
||||
self.dropDb()
|
||||
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -63,14 +63,18 @@ class TBase:
|
|||
# db action
|
||||
#
|
||||
|
||||
def trimDb(self):
|
||||
tdSql.execute(f"trim database {self.db}")
|
||||
def trimDb(self, show = False):
|
||||
tdSql.execute(f"trim database {self.db}", show = show)
|
||||
|
||||
def compactDb(self):
|
||||
tdSql.execute(f"compact database {self.db}")
|
||||
def compactDb(self, show = False):
|
||||
tdSql.execute(f"compact database {self.db}", show = show)
|
||||
|
||||
def flushDb(self, show = False):
|
||||
tdSql.execute(f"flush database {self.db}", show = show)
|
||||
|
||||
def dropDb(self, show = False):
|
||||
tdSql.execute(f"drop database {self.db}", show = show)
|
||||
|
||||
def flushDb(self):
|
||||
tdSql.execute(f"flush database {self.db}")
|
||||
|
||||
#
|
||||
# check db correct
|
||||
|
|
|
|||
|
|
@ -20,11 +20,16 @@ import os
|
|||
import time
|
||||
import datetime
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
# if windows platform return True
|
||||
def isWin():
|
||||
return platform.system().lower() == 'windows'
|
||||
|
||||
#
|
||||
# execute programe
|
||||
#
|
||||
|
||||
# wait util execute file finished
|
||||
def exe(file):
|
||||
return os.system(file)
|
||||
|
|
@ -34,3 +39,19 @@ def exeNoWait(file):
|
|||
print("exe no wait")
|
||||
|
||||
|
||||
# run return output and error
|
||||
def run(command):
|
||||
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
process.wait(3)
|
||||
|
||||
output = process.stdout.read().decode(encoding="gbk")
|
||||
error = process.stderr.read().decode(encoding="gbk")
|
||||
|
||||
return output, error
|
||||
|
||||
|
||||
# return list after run
|
||||
def runRetList(command):
|
||||
lines = []
|
||||
output,error = run(command)
|
||||
return output.splitlines()
|
||||
|
|
|
|||
|
|
@ -51,3 +51,6 @@ def binPath():
|
|||
|
||||
def binFile(filename):
|
||||
return binPath() + filename
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ import os
|
|||
import time
|
||||
import datetime
|
||||
|
||||
from frame.server.dnodes import *
|
||||
|
||||
class srvCtl:
|
||||
def __init__(self):
|
||||
# record server information
|
||||
|
|
@ -24,4 +26,19 @@ class srvCtl:
|
|||
self.mLevel = 0
|
||||
self.mLevelDisk = 0
|
||||
|
||||
#
|
||||
# about path
|
||||
#
|
||||
|
||||
# get cluster root path like /root/TDinternal/sim/
|
||||
def clusterRootPath(self):
|
||||
return tdDnodes.getDnodesRootDir()
|
||||
|
||||
# return dnode data files list
|
||||
def dnodeDataFiles(self, idx):
|
||||
files = []
|
||||
return files
|
||||
|
||||
|
||||
|
||||
sc = srvCtl()
|
||||
|
|
@ -5,10 +5,17 @@
|
|||
#unit-test
|
||||
,,y,unit-test,bash test.sh
|
||||
|
||||
#army-test
|
||||
#
|
||||
# army-test
|
||||
#
|
||||
,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2
|
||||
,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1
|
||||
|
||||
#system test
|
||||
|
||||
|
||||
#
|
||||
# system test
|
||||
#
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py
|
||||
,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py
|
||||
|
|
|
|||
Loading…
Reference in New Issue