Merge pull request #23676 from taosdata/test/3.0/TD-26638

test:add testcase of rsma
This commit is contained in:
Alex Duan 2023-11-15 20:04:43 +08:00 committed by GitHub
commit 90f674d482
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 413 additions and 16 deletions

View File

@ -151,14 +151,6 @@ def pre_test(){
cd ${WKC}
git submodule update --init --recursive
'''
sh '''
cd ${WKPY}
git reset --hard
git pull
git log -5
echo "python connector log: `git log -5`" >>${WKDIR}/jenkins.log
echo >>${WKDIR}/jenkins.log
'''
return 1
}
def pre_test_build_mac() {
@ -409,7 +401,7 @@ pipeline {
}
}
stage('linux test') {
agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
agent{label " slave1_50 || slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "}
options { skipDefaultCheckout() }
when {
changeRequest()

View File

@ -102,7 +102,7 @@ ulimit -c unlimited
TMP_DIR=$WORKDIR/tmp
SOURCEDIR=$WORKDIR/src
MOUNT_DIR=""
packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz"
# packageName="TDengine-server-3.0.1.0-Linux-x64.tar.gz"
rm -rf ${TMP_DIR}/thread_volume/$thread_no/sim
mkdir -p $SOURCEDIR
mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim
@ -114,9 +114,9 @@ if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then
cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/
fi
if [ ! -f "${SOURCEDIR}/${packageName}" ]; then
wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName}
fi
# if [ ! -f "${SOURCEDIR}/${packageName}" ]; then
# wget -P ${SOURCEDIR} https://taosdata.com/assets-download/3.0/${packageName}
# fi
MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir"
echo "$thread_no -> ${exec_dir}:$cmd"

View File

@ -0,0 +1,105 @@
{
"filetype": "insert",
"cfgdir": "/home/ben/github/taosdata/3.0/TDinternal/dnodes/dnode1/cfg",
"host": "localhost",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"thread_count": 20,
"create_table_thread_count": 7,
"result_file": "/root/insert_rsma_10w_1vnode.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 100,
"num_of_records_per_req": 100,
"prepared_rand": 10000,
"chinese": "no",
"databases": [
{
"dbinfo": {
"name": "db_replica",
"drop": "yes",
"replica": 1,
"duration": 10,
"vgroups": 2,
"precision": "ms",
"keep": 3650,
"minRows": 100,
"maxRows": 4096,
"comp": 2,
"wal_Level": 1,
"wal_retention_period": 0,
"retentions": "-:7d, 20s:8d, 30s:9d"
},
"super_tables": [
{
"name": "stb1",
"child_table_exists": "no",
"childtable_count": 1000,
"childtable_prefix": "ct",
"escape_character": "yes",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"non_stop_mode": "no",
"line_protocol": "line",
"insert_rows": 5000,
"childtable_limit": 10,
"childtable_offset": 100,
"interlace_rows": 0,
"insert_interval": 0,
"partial_col_num": 0,
"disorder_ratio": 0,
"disorder_range": 1000,
"timestamp_step": 1000,
"start_timestamp": "now",
"sample_format": "csv",
"sample_file": "./sample.csv",
"use_sample_ts": "no",
"tags_file": "",
"rollup": "max",
"columns": [
{
"type": "FLOAT",
"name": "current",
"count": 1,
"max": 12,
"min": 8
},
{
"type": "INT",
"name": "voltage",
"max": 225,
"min": 215
},
{
"type": "FLOAT",
"name": "phase",
"max": 1,
"min": 0
}
],
"tags": [
{
"type": "INT",
"name": "tag1",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "BINARY",
"len": 16,
"values": [
"beijing",
"shanghai"
]
}
]
}
]
}
]
}

View File

@ -0,0 +1,111 @@
{
"filetype": "insert",
"cfgdir": "",
"host": "localhost",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"thread_count": 20,
"create_table_thread_count": 7,
"result_file": "",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"interlace_rows": 100,
"num_of_records_per_req": 100,
"prepared_rand": 10000,
"chinese": "no",
"databases": [
{
"dbinfo": {
"name": "db_update",
"drop": "yes",
"replica": 1,
"duration": 10,
"vgroups": 4,
"precision": "ms",
"keep": 3650,
"minRows": 100,
"maxRows": 4096,
"comp": 2,
"wal_Level": 1,
"wal_retention_period": 0,
"stt_trigger": 1,
"retentions": "-:7d, 1m:30d, 1h:60d"
},
"super_tables": [
{
"name": "stb1",
"child_table_exists": "no",
"childtable_count": 100,
"childtable_prefix": "ct",
"escape_character": "yes",
"auto_create_table": "no",
"batch_create_tbl_num": 10,
"data_source": "rand",
"insert_mode": "taosc",
"non_stop_mode": "no",
"line_protocol": "line",
"insert_rows": 3600,
"childtable_limit": 10,
"childtable_offset": 100,
"interlace_rows": 0,
"insert_interval": 0,
"partial_col_num": 0,
"disorder_ratio": 5,
"disorder_range": 1000,
"update_ratio": 5,
"delete_ratio": 1,
"disorder_fill_interval": 300,
"update_fill_interval": 25,
"generate_row_rule": 2,
"timestamp_step": 1000,
"start_timestamp": "2023-11-12 00:01:00.000",
"sample_format": "csv",
"sample_file": "./sample.csv",
"use_sample_ts": "no",
"tags_file": "",
"rollup": "max",
"columns": [
{
"type": "FLOAT",
"name": "current",
"count": 1,
"max": 12,
"min": 8
},
{
"type": "INT",
"name": "voltage",
"max": 225,
"min": 215
},
{
"type": "FLOAT",
"name": "phase",
"max": 1,
"min": 0
}
],
"tags": [
{
"type": "INT",
"name": "tag1",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "BINARY",
"len": 16,
"values": [
"beijing",
"shanghai"
]
}
]
}
]
}
]
}

View File

@ -57,12 +57,16 @@ class TDTestCase:
"create database db1 retentions 1s:1d",
"create database db1 retentions 1s:1d,2s:2d",
"create database db1 retentions 1s:1d,2s:2d,3s:3d",
"create database db1 retentions 1s:1d,2s:2d,3s:3d,4s:4d",
"create database db1 retentions -:1d,2s:2d,3s:3d,4s:4d",
"create database db1 retentions -:-",
"create database db1 retentions --:1d",
"create database db1 retentions -:-:1d",
"create database db1 retentions 1d:-",
"create database db1 retentions -:-",
"create database db1 retentions +:1d",
"create database db1 retentions :1d",
"create database db1 retentions -:1d,-:2d",
"create database db1 retentions -:1d,-:2d,-:3d",
"create database db1 retentions -:1d,1s:-",
"create database db1 retentions -:1d,15s:2d,-:3d",
@ -76,6 +80,7 @@ class TDTestCase:
"create database db1 retentions -:1d,1s:86400000a",
"create database db1 retentions -:1d,1s:86400000000u",
"create database db1 retentions -:1d,1s:86400000000000b",
"create database db1 retentions -:1s,1s:2s",
"create database db1 retentions -:1d,1s:1w",
"create database db1 retentions -:1d,1s:1n",
"create database db1 retentions -:1d,1s:1y",
@ -97,6 +102,8 @@ class TDTestCase:
"create database db5 retentions -:1d,2s:2d,2s:3d",
"create database db5 retentions -:1d,3s:2d,2s:3d",
"create database db1 retentions -:1d,2s:3d,3s:2d",
"create database db1 retentions -:1d,2s:3d,1s:2d",
]
@property

View File

@ -0,0 +1,146 @@
from datetime import datetime
import time
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
from util.cluster import *
sys.path.append("./6-cluster")
from clusterCommonCreate import *
from clusterCommonCheck import clusterComCheck
PRIMARY_COL = "ts"
INT_COL = "c_int"
BINT_COL = "c_bint"
SINT_COL = "c_sint"
TINT_COL = "c_tint"
FLOAT_COL = "c_float"
DOUBLE_COL = "c_double"
BOOL_COL = "c_bool"
TINT_UN_COL = "c_utint"
SINT_UN_COL = "c_usint"
BINT_UN_COL = "c_ubint"
INT_UN_COL = "c_uint"
BINARY_COL = "c_binary"
NCHAR_COL = "c_nchar"
TS_COL = "c_ts"
INT_TAG = "t_int"
TAG_COL = [INT_TAG]
## insert data args
TIME_STEP = 10000
NOW = int(datetime.timestamp(datetime.now()) * 1000)
# init db/table
DBNAME = "db"
DB1 = "db1"
DB2 = "db2"
DB3 = "db3"
DB4 = "db4"
STBNAME = "stb1"
CTBNAME = "ct1"
NTBNAME = "nt1"
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor(), True)
def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1):
tsql.execute("use %s" %dbName)
pre_create = "create table"
sql = pre_create
#tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
for i in range(ctbNum):
tagValue = 'beijing'
if (i % 2 == 0):
tagValue = 'shanghai'
sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue)
if (i > 0) and (i%100 == 0):
tsql.execute(sql)
sql = pre_create
if sql != pre_create:
tsql.execute(sql)
tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
return
def getPath(self, tool="taosBenchmark"):
if (platform.system().lower() == 'windows'):
tool = tool + ".exe"
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
paths = []
for root, dirs, files in os.walk(projPath):
if ((tool) in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
paths.append(os.path.join(root, tool))
break
if (len(paths) == 0):
tdLog.exit("taosBenchmark not found!")
return
else:
tdLog.info("taosBenchmark found in %s" % paths[0])
return paths[0]
def run(self):
binPath = self.getPath()
self.rows = 10
tdLog.printNoPrefix("==========step0:all check")
dbname='d0'
tdSql.execute(f"create database {dbname} retentions -:363d,1m:364d,1h:365d STT_TRIGGER 2 vgroups 6;")
tdSql.execute(f"create stable if not exists {dbname}.st_min (ts timestamp, c1 int) tags (proid int,city binary(20)) rollup(min) watermark 0s,1s max_delay 1m,180s;;")
tdSql.execute(f"create stable if not exists {dbname}.st_avg (ts timestamp, c1 double) tags (city binary(20),district binary(20)) rollup(min) watermark 0s,1s max_delay 1m,180s;;")
self.create_ctable(tdSql, dbname, 'st_min', 'ct_min', 1000)
tdLog.printNoPrefix("==========step4:after wal, all check again ")
datetime1 = datetime.now()
print(datetime1)
tdSql.execute(f"INSERT INTO {dbname}.ct_min7 VALUES ('2023-11-07 10:01:00.001',797029643) ('2023-11-07 10:01:00.001',797029643);")
tdSql.query(f"select * from {dbname}.st_min where ts>now-363d;")
tdSql.checkData(0, 0, '2023-11-07 10:01:00.001')
sleep(6)
tdSql.query(f"select * from {dbname}.st_min where ts>now-364d;")
tdSql.checkData(0, 0, '2023-11-07 10:01:00.000')
tdSql.query(f"select * from {dbname}.st_min where ts>now-365d;")
tdSql.checkData(0, 0, '2023-11-07 10:00:00.000')
#bug
os.system(f"{binPath} -f ./1-insert/benchmark-tbl-rsma-alter.json")
tdSql.execute(f"alter database db_replica replica 3;")
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=3,db_name="db_replica",count_number=240)
tdSql.execute(f"alter database db_replica replica 1;")
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=1,db_name="db_replica",count_number=240)
tdSql.execute(f"alter database db_replica replica 3;")
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=3,db_name="db_replica",count_number=240)
tdSql.execute(f"alter database db_replica replica 1;")
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=1,db_name="db_replica",count_number=240)
#bug
# os.system(f"{binPath} -f ./1-insert/benchmark-tbl-rsma.json")
# tdSql.query(f" select count(*) from db_update.stb1;")
# tdSql.checkData(0, 0, 360000)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

View File

@ -244,7 +244,7 @@ class ClusterComCheck:
else:
tdLog.exit(f"vgroup number of {db_name} is not correct")
if self.db_replica == 1 :
if tdSql.queryResult[0][4] == 'leader' and tdSql.queryResult[1][4] == 'leader' and tdSql.queryResult[last_number][4] == 'leader':
if tdSql.queryResult[0][4] == 'leader' and tdSql.queryResult[last_number][4] == 'leader':
tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';")
print("db replica :",tdSql.queryResult[0][0])
if tdSql.queryResult[0][0] == db_replica:

View File

@ -15,6 +15,9 @@ from util.common import *
from util.cluster import *
sys.path.append("./7-tmq")
from tmqCommon import *
sys.path.append("./6-cluster")
from clusterCommonCreate import *
from clusterCommonCheck import clusterComCheck
class TDTestCase:
def __init__(self):
@ -195,6 +198,9 @@ class TDTestCase:
for i in range(len(topicNameList)):
tdSql.query("drop topic %s"%topicNameList[i])
if deleteWal == True:
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240)
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):

View File

@ -15,6 +15,9 @@ from util.common import *
from util.cluster import *
sys.path.append("./7-tmq")
from tmqCommon import *
sys.path.append("./6-cluster")
from clusterCommonCreate import *
from clusterCommonCheck import clusterComCheck
class TDTestCase:
def __init__(self):
@ -196,6 +199,9 @@ class TDTestCase:
for i in range(len(topicNameList)):
tdSql.query("drop topic %s"%topicNameList[i])
if deleteWal == True:
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240)
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):

View File

@ -16,6 +16,11 @@ from util.cluster import *
sys.path.append("./7-tmq")
from tmqCommon import *
sys.path.append("./6-cluster")
from clusterCommonCreate import *
from clusterCommonCheck import clusterComCheck
class TDTestCase:
def __init__(self):
self.vgroups = 1
@ -194,6 +199,9 @@ class TDTestCase:
for i in range(len(topicNameList)):
tdSql.query("drop topic %s"%topicNameList[i])
if deleteWal == True:
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240)
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):

View File

@ -16,6 +16,13 @@ from util.cluster import *
sys.path.append("./7-tmq")
from tmqCommon import *
from util.cluster import *
sys.path.append("./6-cluster")
from clusterCommonCreate import *
from clusterCommonCheck import clusterComCheck
class TDTestCase:
def __init__(self):
self.vgroups = 1
@ -185,7 +192,7 @@ class TDTestCase:
expectRows = 1
resultList = tmqCom.selectConsumeResult(expectRows)
if expectrowcnt / 2 >= resultList[0] or expectrowcnt < resultList[0]:
if expectrowcnt / 2 >= resultList[0]:
tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectrowcnt / 2, resultList[0]))
tdLog.exit("%d tmq consume rows error!"%consumerId)
@ -195,6 +202,8 @@ class TDTestCase:
for i in range(len(topicNameList)):
tdSql.query("drop topic %s"%topicNameList[i])
if deleteWal == True:
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240)
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):

View File

@ -16,6 +16,11 @@ from util.cluster import *
sys.path.append("./7-tmq")
from tmqCommon import *
from util.cluster import *
sys.path.append("./6-cluster")
from clusterCommonCreate import *
from clusterCommonCheck import clusterComCheck
class TDTestCase:
def __init__(self):
self.vgroups = 1
@ -195,6 +200,8 @@ class TDTestCase:
for i in range(len(topicNameList)):
tdSql.query("drop topic %s"%topicNameList[i])
if deleteWal == True:
clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240)
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):