xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #28376 from taosdata/feat/TD-32441-3.0 

feat: support query-QPS new feature
This commit is contained in:
Alex Duan 2024-10-16 18:06:25 +08:00 committed by GitHub
parent bf326e68d9 1d018d0d28
commit b318f12a2a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 88 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tools/auto/stmt2Performance/json/template.json
View File

@ -16,14 +16,14 @@
{
"dbinfo": {
"name": "dbrate",
"drop": "yes",
"vgroups": 2
"vgroups": 1,
"drop": "yes"
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 10,
"childtable_count": 1,
"childtable_prefix": "d",
"insert_mode": "@STMT_MODE",
"interlace_rows": @INTERLACE_MODE,

110
tools/auto/stmt2Performance/stmt2Perf.py
View File

@ -198,16 +198,20 @@ def findContextValue(context, label):
def writeTemplateInfo(resultFile):
# create info
context = readFileContext(templateFile)
context = readFileContext(templateFile)
vgroups = findContextValue(context, "vgroups")
childCount = findContextValue(context, "childtable_count")
insertRows = findContextValue(context, "insert_rows")
line = f"vgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\n\n"
bindVGroup = findContextValue(context, "thread_bind_vgroup")
nThread = findContextValue(context, "thread_count")
if bindVGroup.lower().find("yes") != -1:
nThread = vgroups
line = f"thread_bind_vgroup = {bindVGroup}\nvgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\ninsertThreads = {nThread} \n\n"
print(line)
appendFileContext(resultFile, line)
def totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed):
def totalCompressRate(stmt, interlace, resultFile, spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max, querySpeed):
global Number
# flush
command = 'taos -s "flush database dbrate;"'
@ -220,7 +224,7 @@ def totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed):
# read compress rate
command = 'taos -s "show table distributed dbrate.meters\G;"'
rets = runRetList(command)
print(rets)
#print(rets)
str1 = rets[5]
arr = str1.split(" ")
@ -234,46 +238,88 @@ def totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed):
str2 = arr[6]
pos = str2.find("=[")
rate = str2[pos+2:]
print("rate =" + rate)
# total data file size
#dataSize = getFolderSize(f"{dataDir}/vnode/")
#dataSizeMB = int(dataSize/1024/1024)
# appand to file
# %("No", "stmtMode", "interlaceRows", "spent", "spent-real", "writeSpeed", "write-real", "query-QPS", "dataSize", "rate")
Number += 1
context = "%10s %10s %15s %10s %10s %30s %15s\n"%( Number, stmt, interlace, str(totalSize)+" MB", rate+"%", writeSpeed + " Records/second", querySpeed)
'''
context = "%2s %6s %10s %10s %10s %15s %15s %16s %16s %16s %16s %16s %8s %8s %8s\n"%(
Number, stmt, interlace, spent + "s", spentReal + "s", writeSpeed + " rows/s", writeReal + " rows/s",
min, avg, p90, p99, max,
querySpeed, str(totalSize) + " MB", rate + "%")
'''
context = "%2s %8s %10s %10s %16s %16s %12s %12s %12s %12s %12s %12s %10s %10s %10s\n"%(
Number, stmt, interlace, spent + "s", spentReal + "s", writeSpeed + "r/s", writeReal + "r/s",
min, avg, p90, p99, max + "ms",
querySpeed, str(totalSize) + " MB", rate + "%")
showLog(context)
appendFileContext(resultFile, context)
def cutEnd(line, start, endChar):
pos = line.find(endChar, start)
if pos == -1:
return line[start:]
return line[start : pos]
def findValue(context, pos, key, endChar,command):
pos = context.find(key, pos)
if pos == -1:
print(f"error, run command={command} output not found \"{key}\" keyword. context={context}")
exit(1)
pos += len(key)
value = cutEnd(context, pos, endChar)
return (value, pos)
def testWrite(jsonFile):
command = f"taosBenchmark -f {jsonFile}"
output, context = run(command, 60000)
print(context)
# SUCC: Spent 0.960248 (real 0.947154) seconds to insert rows: 100000 with 1 thread(s) into dbrate 104139.76 (real 105579.45) records/second
# find second real
pos = context.find("(real ")
# spent
key = "Spent "
pos = -1
pos1 = 0
while pos1 != -1: # find last "Spent "
pos1 = context.find(key, pos1)
if pos1 != -1:
pos = pos1 # update last found
pos1 += len(key)
if pos == -1:
print(f"error, run command={command} output not found first \"(real\" keyword. error={context}")
print(f"error, run command={command} output not found \"{key}\" keyword. context={context}")
exit(1)
pos = context.find("(real ", pos + 5)
pos += len(key)
spent = cutEnd(context, pos, ".")
# spent-real
spentReal, pos = findValue(context, pos, "(real ", ".", command)
# writeSpeed
key = "into "
pos = context.find(key, pos)
if pos == -1:
print(f"error, run command={command} output not found second \"(real\" keyword. error={context}")
exit(1)
pos += 5
length = len(context)
while pos < length and context[pos] == ' ':
pos += 1
end = context.find(".", pos)
if end == -1:
print(f"error, run command={command} output not found second \".\" keyword. error={context}")
print(f"error, run command={command} output not found \"{key}\" keyword. context={context}")
exit(1)
pos += len(key)
writeSpeed, pos = findValue(context, pos, " ", ".", command)
# writeReal
writeReal, pos = findValue(context, pos, "(real ", ".", command)
speed = context[pos: end]
#print(f"write pos ={pos} end={end} speed={speed}\n output={context} \n")
return speed
# delay
min, pos = findValue(context, pos, "min: ", ",", command)
avg, pos = findValue(context, pos, "avg: ", ",", command)
p90, pos = findValue(context, pos, "p90: ", ",", command)
p99, pos = findValue(context, pos, "p99: ", ",", command)
max, pos = findValue(context, pos, "max: ", "ms", command)
return (spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max)
def testQuery():
command = f"taosBenchmark -f json/query.json"
@ -308,13 +354,13 @@ def doTest(stmt, interlace, resultFile):
# run taosBenchmark
t1 = time.time()
writeSpeed = testWrite(jsonFile)
spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max = testWrite(jsonFile)
t2 = time.time()
# total write speed
querySpeed = testQuery()
# total compress rate
totalCompressRate(stmt, interlace, resultFile, writeSpeed, querySpeed)
totalCompressRate(stmt, interlace, resultFile, spent, spentReal, writeSpeed, writeReal, min, avg, p90, p99, max, querySpeed)
def main():
@ -333,7 +379,17 @@ def main():
# json info
writeTemplateInfo(resultFile)
# head
context = "\n%10s %10s %15s %10s %10s %30s %15s\n"%("No", "stmtMode", "interlaceRows", "dataSize", "rate", "writeSpeed", "query-QPS")
'''
context = "%3s %8s %10s %10s %10s %15s %15s %10s %10s %10s %10s %10s %8s %8s %8s\n"%(
"No", "stmtMode", "interlace", "spent", "spent-real", "writeSpeed", "write-real",
"min", "avg", "p90", "p99", "max",
"query-QPS", "dataSize", "rate")
'''
context = "%2s %8s %10s %10s %16s %16s %12s %12s %12s %12s %12s %12s %10s %10s %10s\n"%(
"No", "stmtMode", "interlace", "spent", "spent-real", "writeSpeed", "write-real",
"min", "avg", "p90", "p99", "max",
"query-QPS", "dataSize", "rate")
appendFileContext(resultFile, context)

9
tools/auto/testCompression/testCompression.py
View File

@ -134,8 +134,6 @@ def getMatch(datatype, algo):
def generateJsonFile(algo):
print(f"doTest algo: {algo} \n")
# replace datatype
context = readFileContext(templateFile)
# replace compress
@ -192,8 +190,6 @@ def findContextValue(context, label):
ends = [',','}',']', 0]
while context[end] not in ends:
end += 1
print(f"start = {start} end={end}\n")
return context[start:end]
@ -281,10 +277,10 @@ def testQuery():
# INFO: Spend 6.7350 second completed total queries: 10, the QPS of all threads: 1.485
speed = None
for i in range(20, len(lines)):
for i in range(0, len(lines)):
# find second real
context = lines[i]
pos = context.find("the QPS of all threads:")
context = lines[26]
if pos == -1 :
continue
pos += 24
@ -302,7 +298,6 @@ def doTest(algo, resultFile):
print(f"doTest algo: {algo} \n")
#cleanAndStartTaosd()
# json
jsonFile = generateJsonFile(algo)