xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-5007]<fix>: use build directory's taosdump to test. (#6710) 

btw, solve two taosdemo test cases too.
This commit is contained in:
Shuduo Sang 2021-07-01 23:28:19 +08:00 committed by GitHub
parent 02051bf8d8
commit 88cc5e5e87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 127 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
tests/pytest/tools/taosdemoPerformance.py
View File

@ -17,10 +17,11 @@ import argparse
import os.path import os.path
import json import json
class taosdemoPerformace: class taosdemoPerformace:
def __init__(self, commitID, dbName): def __init__(self, commitID, dbName):
self.commitID = commitID self.commitID = commitID
self.dbName = dbName self.dbName = dbName
self.host = "127.0.0.1" self.host = "127.0.0.1"
self.user = "root" self.user = "root"
self.password = "taosdata" self.password = "taosdata"
@ -30,8 +31,8 @@ class taosdemoPerformace:
self.user, self.user,
self.password, self.password,
self.config) self.config)
self.insertDB = "insertDB"; self.insertDB = "insertDB"
def generateJson(self): def generateJson(self):
db = { db = {
"name": "%s" % self.insertDB, "name": "%s" % self.insertDB,
@ -41,7 +42,7 @@ class taosdemoPerformace:
stb = { stb = {
"name": "meters", "name": "meters",
"child_table_exists":"no", "child_table_exists": "no",
"childtable_count": 10000, "childtable_count": 10000,
"childtable_prefix": "stb_", "childtable_prefix": "stb_",
"auto_create_table": "no", "auto_create_table": "no",
@ -57,12 +58,12 @@ class taosdemoPerformace:
"start_timestamp": "2020-10-01 00:00:00.000", "start_timestamp": "2020-10-01 00:00:00.000",
"sample_format": "csv", "sample_format": "csv",
"sample_file": "./sample.csv", "sample_file": "./sample.csv",
"tags_file": "", "tags_file": "",
"columns": [ "columns": [
{"type": "INT", "count": 4} {"type": "INT", "count": 4}
], ],
"tags": [ "tags": [
{"type": "INT", "count":1}, {"type": "INT", "count": 1},
{"type": "BINARY", "len": 16} {"type": "BINARY", "len": 16}
] ]
} }
@ -88,7 +89,7 @@ class taosdemoPerformace:
"confirm_parameter_prompt": "no", "confirm_parameter_prompt": "no",
"insert_interval": 0, "insert_interval": 0,
"num_of_records_per_req": 30000, "num_of_records_per_req": 30000,
"databases": [db] "databases": [db]
} }
insert_json_file = f"/tmp/insert.json" insert_json_file = f"/tmp/insert.json"
@ -103,24 +104,56 @@ class taosdemoPerformace:
cmd.close() cmd.close()
return output return output
def insertData(self): def getBuildPath(self):
os.system("taosdemo -f %s > taosdemoperf.txt 2>&1" % self.generateJson()) selfPath = os.path.dirname(os.path.realpath(__file__))
self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'")
self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") if ("community" in selfPath):
self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosdemo" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def insertData(self):
tdSql.prepare()
buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosdemo not found!")
else:
tdLog.info("taosdemo found in %s" % buildPath)
binPath = buildPath + "/build/bin/"
os.system(
"%staosdemo -f %s > taosdemoperf.txt 2>&1" %
(binPath, self.generateJson()))
self.createTableTime = self.getCMDOutput(
"grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'")
self.insertRecordsTime = self.getCMDOutput(
"grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'")
self.recordsPerSecond = self.getCMDOutput(
"grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'")
self.commitID = self.getCMDOutput("git rev-parse --short HEAD") self.commitID = self.getCMDOutput("git rev-parse --short HEAD")
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'") delay = self.getCMDOutput(
"grep 'delay' taosdemoperf.txt | awk '{print $4}'")
self.avgDelay = delay[:-4] self.avgDelay = delay[:-4]
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'") delay = self.getCMDOutput(
"grep 'delay' taosdemoperf.txt | awk '{print $6}'")
self.maxDelay = delay[:-4] self.maxDelay = delay[:-4]
delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'") delay = self.getCMDOutput(
"grep 'delay' taosdemoperf.txt | awk '{print $8}'")
self.minDelay = delay[:-3] self.minDelay = delay[:-3]
os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt") os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt")
def createTablesAndStoreData(self): def createTablesAndStoreData(self):
cursor = self.conn.cursor() cursor = self.conn.cursor()
cursor.execute("create database if not exists %s" % self.dbName) cursor.execute("create database if not exists %s" % self.dbName)
cursor.execute("use %s" % self.dbName) cursor.execute("use %s" % self.dbName)
cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)")
@ -130,13 +163,21 @@ class taosdemoPerformace:
print("records per second: %f" % float(self.recordsPerSecond)) print("records per second: %f" % float(self.recordsPerSecond))
print("avg delay: %f" % float(self.avgDelay)) print("avg delay: %f" % float(self.avgDelay))
print("max delay: %f" % float(self.maxDelay)) print("max delay: %f" % float(self.maxDelay))
print("min delay: %f" % float(self.minDelay)) print("min delay: %f" % float(self.minDelay))
cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % cursor.execute(
(float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay))) "insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" %
(float(
self.createTableTime), float(
self.insertRecordsTime), float(
self.recordsPerSecond), self.commitID, float(
self.avgDelay), float(
self.maxDelay), float(
self.minDelay)))
cursor.execute("drop database if exists %s" % self.insertDB) cursor.execute("drop database if exists %s" % self.insertDB)
cursor.close() cursor.close()
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument( parser.add_argument(
@ -155,6 +196,6 @@ if __name__ == '__main__':
args = parser.parse_args() args = parser.parse_args()
perftest = taosdemoPerformace(args.commit_id, args.database_name) perftest = taosdemoPerformace(args.commit_id, args.database_name)
perftest.insertData() perftest.insertData()
perftest.createTablesAndStoreData() perftest.createTablesAndStoreData()

6
tests/pytest/tools/taosdemoTest.py
View File

@ -36,7 +36,7 @@ class TDTestCase:
projPath = selfPath[:selfPath.find("tests")] projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath): for root, dirs, files in os.walk(projPath):
if ("taosd" in files): if ("taosdemo" in files):
rootRealPath = os.path.dirname(os.path.realpath(root)) rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath): if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")] buildPath = root[:len(root) - len("/build/bin")]
@ -47,9 +47,9 @@ class TDTestCase:
tdSql.prepare() tdSql.prepare()
buildPath = self.getBuildPath() buildPath = self.getBuildPath()
if (buildPath == ""): if (buildPath == ""):
tdLog.exit("taosd not found!") tdLog.exit("taosdemo not found!")
else: else:
tdLog.info("taosd found in %s" % buildPath) tdLog.info("taosdemo found in %s" % buildPath)
binPath = buildPath + "/build/bin/" binPath = buildPath + "/build/bin/"
os.system("%staosdemo -y -t %d -n %d" % os.system("%staosdemo -y -t %d -n %d" %
(binPath, self.numberOfTables, self.numberOfRecords)) (binPath, self.numberOfTables, self.numberOfRecords))

94
tests/pytest/tools/taosdumpTest.py
View File

@ -23,51 +23,79 @@ class TDTestCase:
def init(self, conn, logSql): def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__) tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql) tdSql.init(conn.cursor(), logSql)
self.ts = 1538548685000 self.ts = 1538548685000
self.numberOfTables = 10000 self.numberOfTables = 10000
self.numberOfRecords = 100 self.numberOfRecords = 100
def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__))
if ("community" in selfPath):
projPath = selfPath[:selfPath.find("community")]
else:
projPath = selfPath[:selfPath.find("tests")]
for root, dirs, files in os.walk(projPath):
if ("taosdump" in files):
rootRealPath = os.path.dirname(os.path.realpath(root))
if ("packaging" not in rootRealPath):
buildPath = root[:len(root) - len("/build/bin")]
break
return buildPath
def run(self): def run(self):
if not os.path.exists("/taosdumptest/tmp1"): if not os.path.exists("./taosdumptest/tmp1"):
os.makedirs("/taosdumptest/tmp1") os.makedirs("./taosdumptest/tmp1")
else: else:
print("目录存在") print("目录存在")
if not os.path.exists("/taosdumptest/tmp2"):
os.makedirs("/taosdumptest/tmp2") if not os.path.exists("./taosdumptest/tmp2"):
os.makedirs("./taosdumptest/tmp2")
tdSql.execute("drop database if exists db") tdSql.execute("drop database if exists db")
tdSql.execute("create database db days 11 keep 3649 blocks 8 ") tdSql.execute("create database db days 11 keep 3649 blocks 8 ")
tdSql.execute("create database db1 days 12 keep 3640 blocks 7 ") tdSql.execute("create database db1 days 12 keep 3640 blocks 7 ")
tdSql.execute("use db") tdSql.execute("use db")
tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") tdSql.execute(
tdSql.execute("create table t1 using st tags(1, 'beijing')") "create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.execute("create table t1 using st tags(1, 'beijing')")
sql = "insert into t1 values" sql = "insert into t1 values"
currts = self.ts currts = self.ts
for i in range(100): for i in range(100):
sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
tdSql.execute(sql) tdSql.execute(sql)
tdSql.execute("create table t2 using st tags(2, 'shanghai')") tdSql.execute("create table t2 using st tags(2, 'shanghai')")
sql = "insert into t2 values" sql = "insert into t2 values"
currts = self.ts currts = self.ts
for i in range(100): for i in range(100):
sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100)
tdSql.execute(sql) tdSql.execute(sql)
os.system("taosdump --databases db -o /taosdumptest/tmp1")
os.system("taosdump --databases db1 -N -o /taosdumptest/tmp2") buildPath = self.getBuildPath()
if (buildPath == ""):
tdLog.exit("taosdump not found!")
else:
tdLog.info("taosdump found in %s" % buildPath)
binPath = buildPath + "/build/bin/"
os.system("%staosdump --databases db -o ./taosdumptest/tmp1" % binPath)
os.system(
"%staosdump --databases db1 -N -o ./taosdumptest/tmp2" %
binPath)
tdSql.execute("drop database db") tdSql.execute("drop database db")
tdSql.execute("drop database db1") tdSql.execute("drop database db1")
tdSql.query("show databases") tdSql.query("show databases")
tdSql.checkRows(0) tdSql.checkRows(0)
os.system("taosdump -i /taosdumptest/tmp1") os.system("%staosdump -i ./taosdumptest/tmp1" % binPath)
os.system("taosdump -i /taosdumptest/tmp2") os.system("%staosdump -i ./taosdumptest/tmp2" % binPath)
tdSql.execute("use db") tdSql.execute("use db")
tdSql.query("show databases") tdSql.query("show databases")
tdSql.checkRows(2) tdSql.checkRows(2)
dbresult = tdSql.queryResult dbresult = tdSql.queryResult
# 6--days,7--keep0,keep1,keep, 12--block, # 6--days,7--keep0,keep1,keep, 12--block,
for i in range(len(dbresult)): for i in range(len(dbresult)):
if dbresult[i][0] == 'db': if dbresult[i][0] == 'db':
print(dbresult[i]) print(dbresult[i])
print(type(dbresult[i][6])) print(type(dbresult[i][6]))
@ -102,35 +130,37 @@ class TDTestCase:
tdSql.checkData(i, 1, i) tdSql.checkData(i, 1, i)
tdSql.checkData(i, 2, "nchar%d" % i) tdSql.checkData(i, 2, "nchar%d" % i)
# drop all databases，boundary value testing. length(databasename)<=32;length(tablesname)<=192 # drop all databases，boundary value testing.
# length(databasename)<=32;length(tablesname)<=192
tdSql.execute("drop database db") tdSql.execute("drop database db")
tdSql.execute("drop database db1") tdSql.execute("drop database db1")
os.system("rm -rf /taosdumptest/tmp1") os.system("rm -rf ./taosdumptest/tmp1")
os.system("rm -rf /taosdumptest/tmp2") os.system("rm -rf ./taosdumptest/tmp2")
os.makedirs("/taosdumptest/tmp1") os.makedirs("./taosdumptest/tmp1")
tdSql.execute("create database db12312313231231321312312312_323") tdSql.execute("create database db12312313231231321312312312_323")
tdSql.error("create database db12312313231231321312312312_3231") tdSql.error("create database db12312313231231321312312312_3231")
tdSql.execute("use db12312313231231321312312312_323") tdSql.execute("use db12312313231231321312312312_323")
tdSql.execute("create stable st12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") tdSql.execute("create stable st12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.error("create stable st_12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") tdSql.error("create stable st_12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.execute("create stable st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") tdSql.execute(
"create stable st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.error("create stable st1(ts timestamp, c1 int, col2_012345678901234567890123456789012345678901234567890123456789 nchar(10)) tags(t1 int, t2 binary(10))") tdSql.error("create stable st1(ts timestamp, c1 int, col2_012345678901234567890123456789012345678901234567890123456789 nchar(10)) tags(t1 int, t2 binary(10))")
tdSql.execute("select * from db12312313231231321312312312_323.st12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9") tdSql.execute("select * from db12312313231231321312312312_323.st12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9")
tdSql.error("create table t0_12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9 using st tags(1, 'beijing')") tdSql.error("create table t0_12345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678912345678_9 using st tags(1, 'beijing')")
tdSql.query("show stables") tdSql.query("show stables")
tdSql.checkRows(2) tdSql.checkRows(2)
os.system("taosdump --databases db12312313231231321312312312_323 -o /taosdumptest/tmp1") os.system(
tdSql.execute("drop database db12312313231231321312312312_323") "%staosdump --databases db12312313231231321312312312_323 -o ./taosdumptest/tmp1" % binPath)
os.system("taosdump -i /taosdumptest/tmp1") tdSql.execute("drop database db12312313231231321312312312_323")
os.system("%staosdump -i ./taosdumptest/tmp1" % binPath)
tdSql.execute("use db12312313231231321312312312_323") tdSql.execute("use db12312313231231321312312312_323")
tdSql.query("show stables") tdSql.query("show stables")
tdSql.checkRows(2) tdSql.checkRows(2)
os.system("rm -rf /taosdumptest/tmp1") os.system("rm -rf ./taosdumptest/tmp1")
os.system("rm -rf /taosdumptest/tmp2") os.system("rm -rf ./taosdumptest/tmp2")
os.system("rm -rf ./dump_result.txt") os.system("rm -rf ./dump_result.txt")
os.system("rm -rf ./db.csv") os.system("rm -rf ./db.csv")
def stop(self): def stop(self):
tdSql.close() tdSql.close()
@ -138,4 +168,4 @@ class TDTestCase:
tdCases.addWindows(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase())