xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

test: add taosbenchmark mix and stt case to ci

This commit is contained in:
Alex Duan 2023-07-07 16:04:35 +08:00
parent db63b43622
commit aa2a5ef0dd
5 changed files with 368 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/parallel_test/cases.task
View File

@ -755,6 +755,8 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py
,,y,system-test,./pytest.sh python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3
,,y,system-test,./pytest.sh python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3
#tsim test
,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim

102
tests/system-test/5-taos-tools/taosbenchmark/insertMix.py Normal file
View File

@ -0,0 +1,102 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import subprocess
import time
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def caseDescription(self):
"""
[TD-13823] taosBenchmark test cases
"""
return
def init(self, conn, logSql, replicaVar=1):
# comment off by Shuduo for CI self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def getPath(self, tool="taosBenchmark"):
if (platform.system().lower() == 'windows'):
tool = tool + ".exe"
selfPath = os.path.dirname(os.path.realpath(__file__))
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
else:
projPath = selfPath[: selfPath.find("tests")]
paths = []
for root, dirs, files in os.walk(projPath):
if (tool) in files:
rootRealPath = os.path.dirname(os.path.realpath(root))
if "packaging" not in rootRealPath:
paths.append(os.path.join(root, tool))
break
if len(paths) == 0:
tdLog.exit("taosBenchmark not found!")
return
else:
tdLog.info("taosBenchmark found in %s" % paths[0])
return paths[0]
def checkDataCorrect(self):
sql = "select count(*) from meters"
tdSql.query(sql)
allCnt = tdSql.getData(0, 0)
if allCnt < 2000000:
tdLog.exit(f"taosbenchmark insert row small. row count={allCnt} sql={sql}")
return
# group by 10 child table
rowCnt = tdSql.query("select count(*),tbname from meters group by tbname")
tdSql.checkRows(10)
# interval
sql = "select count(*),max(ic),min(dc),last(*) from meters interval(1s)"
rowCnt = tdSql.query(sql)
if rowCnt < 10:
tdLog.exit(f"taosbenchmark interval(1s) count small. row cout={rowCnt} sql={sql}")
return
# nest query
tdSql.query("select count(*) from (select * from meters order by ts desc)")
tdSql.checkData(0, 0, allCnt)
def run(self):
binPath = self.getPath()
cmd = "%s -f ./5-taos-tools/taosbenchmark/json/insertMix.json" % binPath
tdLog.info("%s" % cmd)
errcode = os.system("%s" % cmd)
if errcode != 0:
tdLog.exit(f"execute taosBenchmark ret error code={errcode}")
return
tdSql.execute("use mixdb")
self.checkDataCorrect()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

81
tests/system-test/5-taos-tools/taosbenchmark/json/insertMix.json Normal file
View File

@ -0,0 +1,81 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 3000,
"thread_count": 10,
"create_table_thread_count": 2,
"result_file": "./insert_res_mix.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"check_sql": "yes",
"continue_if_fail": "no",
"databases": [
{
"dbinfo": {
"name": "mixdb",
"drop": "yes",
"vgroups": 6,
"replica": 3,
"precision": "ms",
"keep": 3650,
"minRows": 100,
"maxRows": 4096
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 10,
"insert_rows": 300000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"insert_interval": 0,
"timestamp_step": 100,
"start_timestamp":1500000000000,
"disorder_ratio": 10,
"update_ratio": 5,
"delete_ratio": 1,
"disorder_fill_interval": 300,
"update_fill_interval": 25,
"generate_row_rule": 2,
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 1, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0 },
{ "type": "int", "name": "ic", "max": 100, "min": 0 },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 100, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0 },
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{
"type": "tinyint",
"name": "groupid",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "binary",
"len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

81
tests/system-test/5-taos-tools/taosbenchmark/json/stt.json Normal file
View File

@ -0,0 +1,81 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"connection_pool_size": 8,
"num_of_records_per_req": 3000,
"thread_count": 20,
"create_table_thread_count": 5,
"result_file": "./insert_res_wal.txt",
"confirm_parameter_prompt": "no",
"insert_interval": 0,
"databases": [
{
"dbinfo": {
"name": "db",
"drop": "yes",
"flush_each_batch": "yes",
"vgroups": 2,
"replica": 1,
"precision": "ms",
"keep": 3650,
"minRows": 100,
"maxRows": 4096
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 1000,
"insert_rows": 2850,
"childtable_prefix": "d",
"insert_mode": "taosc",
"insert_interval": 0,
"timestamp_step": 10,
"disorder_ratio": 10,
"update_ratio": 5,
"delete_ratio": 1,
"disorder_fill_interval": 30,
"update_fill_interval": 25,
"generate_row_rule": 2,
"start_timestamp":"2022-01-01 10:00:00",
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 1, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0 },
{ "type": "int", "name": "ic", "max": 100, "min": 0 },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 100, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0 },
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{
"type": "tinyint",
"name": "groupid",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "binary",
"len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

102
tests/system-test/5-taos-tools/taosbenchmark/stt.py Normal file
View File

@ -0,0 +1,102 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import os
import subprocess
import time
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def caseDescription(self):
"""
[TD-13823] taosBenchmark test cases
"""
return
def init(self, conn, logSql, replicaVar=1):
# comment off by Shuduo for CI self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def getPath(self, tool="taosBenchmark"):
if (platform.system().lower() == 'windows'):
tool = tool + ".exe"
selfPath = os.path.dirname(os.path.realpath(__file__))
if "community" in selfPath:
projPath = selfPath[: selfPath.find("community")]
else:
projPath = selfPath[: selfPath.find("tests")]
paths = []
for root, dirs, files in os.walk(projPath):
if (tool) in files:
rootRealPath = os.path.dirname(os.path.realpath(root))
if "packaging" not in rootRealPath:
paths.append(os.path.join(root, tool))
break
if len(paths) == 0:
tdLog.exit("taosBenchmark not found!")
return
else:
tdLog.info("taosBenchmark found in %s" % paths[0])
return paths[0]
def checkDataCorrect(self):
sql = "select count(*) from meters"
tdSql.query(sql)
allCnt = tdSql.getData(0, 0)
if allCnt < 2000000:
tdLog.exit(f"taosbenchmark insert row small. row count={allCnt} sql={sql}")
return
# group by 10 child table
rowCnt = tdSql.query("select count(*),tbname from meters group by tbname")
tdSql.checkRows(1000)
# interval
sql = "select count(*),max(ic),min(dc),last(*) from meters interval(1s)"
rowCnt = tdSql.query(sql)
if rowCnt < 10:
tdLog.exit(f"taosbenchmark interval(1s) count small. row cout={rowCnt} sql={sql}")
return
# nest query
tdSql.query("select count(*) from (select * from meters order by ts desc)")
tdSql.checkData(0, 0, allCnt)
def run(self):
binPath = self.getPath()
cmd = "%s -f ./5-taos-tools/taosbenchmark/json/stt.json" % binPath
tdLog.info("%s" % cmd)
errcode = os.system("%s" % cmd)
if errcode != 0:
tdLog.exit(f"execute taosBenchmark ret error code={errcode}")
return
tdSql.execute("use db")
self.checkDataCorrect()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())