diff --git a/tools/auto/testCompression/json/query.json b/tools/auto/testCompression/json/query.json new file mode 100644 index 0000000000..e810c1009f --- /dev/null +++ b/tools/auto/testCompression/json/query.json @@ -0,0 +1,23 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "continue_if_fail": "yes", + "databases": "dbrate", + "query_times": 20, + "query_mode": "taosc", + "specified_table_query": { + "query_interval": 0, + "concurrent": 10, + "sqls": [ + { + "sql": "select * from meters", + "result": "./query_res0.txt" + } + ] + } +} diff --git a/tools/auto/testCompression/testCompression.py b/tools/auto/testCompression/testCompression.py index 768fa87f71..43d7a9ae91 100644 --- a/tools/auto/testCompression/testCompression.py +++ b/tools/auto/testCompression/testCompression.py @@ -48,9 +48,13 @@ def run(command, timeout = 60, show=True): return output, error # return list after run -def runRetList(command, timeout=10): +def runRetList(command, timeout=10, first=True): output,error = run(command, timeout) - return output.splitlines() + if first: + return output.splitlines() + else: + return error.splitlines() + def readFileContext(filename): file = open(filename) @@ -204,7 +208,7 @@ def writeTemplateInfo(resultFile): appendFileContext(resultFile, line) -def totalCompressRate(algo, resultFile, writeSecond): +def totalCompressRate(algo, resultFile, writeSpeed, querySpeed): global Number # flush command = 'taos -s "flush database dbrate;"' @@ -239,10 +243,51 @@ def totalCompressRate(algo, resultFile, writeSecond): # appand to file Number += 1 - context = "%10s %10s %10s %10s %10s\n"%( Number, algo, str(totalSize)+" MB", rate+"%", writeSecond + " s") + context = "%10s %10s %10s %10s %30s %15s\n"%( Number, algo, str(totalSize)+" MB", rate+"%", writeSpeed + " Records/second", querySpeed) showLog(context) appendFileContext(resultFile, context) +def testWrite(jsonFile): + command = f"taosBenchmark -f {jsonFile}" + lines = runRetList(command, 60000, False) + context = lines[3] + # SUCC: Spent 0.960248 (real 0.947154) seconds to insert rows: 100000 with 1 thread(s) into dbrate 104139.76 (real 105579.45) records/second + + # find second real + pos = context.find("(real ") + if pos == -1: + print(f"error, run command={command} output not found first \"(real\" keyword. error={context}") + exit(1) + pos = context.find("(real ", pos + 5) + if pos == -1: + print(f"error, run command={command} output not found second \"(real\" keyword. error={context}") + exit(1) + + pos += 5 + length = len(context) + while pos < length and context[pos] == ' ': + pos += 1 + end = context.find(") records/second", pos) + if end == -1: + print(f"error, run command={command} output not found second \") records/second\" keyword. error={context}") + exit(1) + + speed = context[pos: end] + #print(f"write pos ={pos} end={end} speed={speed}\n output={context} \n") + return speed + +def testQuery(): + command = f"taosBenchmark -f json/query.json" + lines = runRetList(command, 60000) + # INFO: Spend 6.7350 second completed total queries: 10, the QPS of all threads: 1.485 + context = lines[26] + + # find second real + pos = context.find("the QPS of all threads:") + pos += 24 + speed = context[pos:] + #print(f"query pos ={pos} speed={speed}\n output={context} \n") + return speed def doTest(algo, resultFile): print(f"doTest algo: {algo} \n") @@ -254,16 +299,20 @@ def doTest(algo, resultFile): # run taosBenchmark t1 = time.time() - exec(f"taosBenchmark -f {jsonFile}") + writeSpeed = testWrite(jsonFile) t2 = time.time() + # total write speed + querySpeed = testQuery() # total compress rate - totalCompressRate(algo, resultFile, str(int(t2-t1))) + totalCompressRate(algo, resultFile, writeSpeed, querySpeed) + def main(): # test compress method algos = ["lz4", "zlib", "zstd", "xz", "disabled"] + #algos = ["lz4"] # record result resultFile = "./result.txt" @@ -275,7 +324,7 @@ def main(): # json info writeTemplateInfo(resultFile) # head - context = "\n%10s %10s %10s %10s %10s\n"%("No", "compress", "dataSize", "rate", "insertSeconds") + context = "\n%10s %10s %10s %10s %30s %15s\n"%("No", "compress", "dataSize", "rate", "writeSpeed", "query-QPS") appendFileContext(resultFile, context)