diff --git a/tools/auto/testCompression/json/d0_10w.csv.zip b/tools/auto/testCompression/json/d0_10w.csv.zip new file mode 100644 index 0000000000..d7dcea6cd5 Binary files /dev/null and b/tools/auto/testCompression/json/d0_10w.csv.zip differ diff --git a/tools/auto/testCompression/json/generate.json b/tools/auto/testCompression/json/generate.json new file mode 100644 index 0000000000..1394dfc28d --- /dev/null +++ b/tools/auto/testCompression/json/generate.json @@ -0,0 +1,64 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "num_of_records_per_req": 500, + "thread_count": 4, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "vgroups": 2 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 1, + "insert_rows": 100000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1, + "start_timestamp":1700000000000, + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc", "max": 1, "min": 0 }, + { "type": "double", "name": "dc", "max": 1, "min": 0 }, + { "type": "tinyint", "name": "ti", "max": 100, "min": 0 }, + { "type": "smallint", "name": "si", "max": 100, "min": 0 }, + { "type": "int", "name": "ic", "max": 100, "min": 0 }, + { "type": "bigint", "name": "bi", "max": 100, "min": 0 }, + { "type": "utinyint", "name": "uti", "max": 100, "min": 0 }, + { "type": "usmallint", "name": "usi", "max": 100, "min": 0 }, + { "type": "uint", "name": "ui", "max": 100, "min": 0 }, + { "type": "ubigint", "name": "ubi", "max": 100, "min": 0 }, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + { + "type": "tinyint", + "name": "groupid", + "max": 10, + "min": 1 + }, + { + "name": "location", + "type": "binary", + "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tools/auto/testCompression/json/template.json b/tools/auto/testCompression/json/template.json new file mode 100644 index 0000000000..1be39d21a1 --- /dev/null +++ b/tools/auto/testCompression/json/template.json @@ -0,0 +1,59 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 1, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "num_of_records_per_req": 2000, + "prepared_rand": 100000, + "escape_character": "yes", + "databases": [ + { + "dbinfo": { + "name": "dbrate", + "drop": "yes", + "vgroups": 1, + "stt_trigger": 1, + "wal_retention_size": 1, + "wal_retention_period": 1 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 1, + "childtable_prefix": "d", + "data_source": "sample", + "insert_mode": "taosc", + "insert_rows": 100000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_file": "./json/d0_10w.csv", + "use_sample_ts": "yes", + "columns": [ + { "type": "bool", "name": "bc", "compress":"@COMPRESS"}, + { "type": "float", "name": "fc", "compress":"@COMPRESS"}, + { "type": "double", "name": "dc", "compress":"@COMPRESS"}, + { "type": "tinyint", "name": "ti", "compress":"@COMPRESS"}, + { "type": "smallint", "name": "si", "compress":"@COMPRESS"}, + { "type": "int", "name": "ic", "compress":"@COMPRESS"}, + { "type": "bigint", "name": "bi", "compress":"@COMPRESS"}, + { "type": "utinyint", "name": "uti","compress":"@COMPRESS"}, + { "type": "usmallint", "name": "usi","compress":"@COMPRESS"}, + { "type": "uint", "name": "ui", "compress":"@COMPRESS"}, + { "type": "ubigint", "name": "ubi","compress":"@COMPRESS"}, + { "type": "binary", "name": "bin", "len": 32, "compress":"@COMPRESS"}, + { "type": "nchar", "name": "nch", "len": 64, "compress":"@COMPRESS"} + ], + "tags": [ + {"type": "TINYINT", "name": "groupid", "max": 10, "min": 1} + ] + } + ] + } + ] +} diff --git a/tools/auto/testCompression/testCompression.py b/tools/auto/testCompression/testCompression.py new file mode 100644 index 0000000000..768fa87f71 --- /dev/null +++ b/tools/auto/testCompression/testCompression.py @@ -0,0 +1,297 @@ +import taos +import sys +import os +import subprocess +import time +import random +import json +from datetime import datetime + + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +dataDir = "/var/lib/taos/" +templateFile = "json/template.json" +Number = 0 +resultContext = "" + + +def showLog(str): + print(str) + +def exec(command, show=True): + if(show): + print(f"exec {command}\n") + return os.system(command) + +# run return output and error +def run(command, timeout = 60, show=True): + if(show): + print(f"run {command} timeout={timeout}s\n") + + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait(timeout) + + output = process.stdout.read().decode(encoding="gbk") + error = process.stderr.read().decode(encoding="gbk") + + return output, error + +# return list after run +def runRetList(command, timeout=10): + output,error = run(command, timeout) + return output.splitlines() + +def readFileContext(filename): + file = open(filename) + context = file.read() + file.close() + return context + + +def writeFileContext(filename, context): + file = open(filename, "w") + file.write(context) + file.close() + +def appendFileContext(filename, context): + global resultContext + resultContext += context + try: + file = open(filename, "a") + wsize = file.write(context) + file.close() + except: + print(f"appand file error context={context} .") + +def getFolderSize(folder): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(folder): + for filename in filenames: + filepath = os.path.join(dirpath, filename) + total_size += os.path.getsize(filepath) + return total_size + +def waitClusterAlive(loop): + for i in range(loop): + command = 'taos -s "show cluster alive\G;" ' + out,err = run(command) + print(out) + if out.find("status: 1") >= 0: + showLog(f" i={i} wait cluster alive ok.\n") + return True + + showLog(f" i={i} wait cluster alive ...\n") + time.sleep(1) + + showLog(f" i={i} wait cluster alive failed.\n") + return False + +def waitCompactFinish(loop): + for i in range(loop): + command = 'taos -s "show compacts;" ' + out,err = run(command) + if out.find("Query OK, 0 row(s) in set") >= 0: + showLog(f" i={i} wait compact finish ok\n") + return True + + showLog(f" i={i} wait compact ...\n") + time.sleep(1) + + showLog(f" i={i} wait compact failed.\n") + return False + + +def getTypeName(datatype): + str1 = datatype.split(",")[0] + str2 = str1.split(":")[1] + str3 = str2.replace('"','').replace(' ','') + return str3 + + +def getMatch(datatype, algo): + if algo == "tsz": + if datatype == "float" or datatype == "double": + return True + else: + return False + else: + return True + + +def generateJsonFile(algo): + print(f"doTest algo: {algo} \n") + + # replace datatype + context = readFileContext(templateFile) + # replace compress + context = context.replace("@COMPRESS", algo) + + # write to file + fileName = f"json/test_{algo}.json" + if os.path.exists(fileName): + os.remove(fileName) + writeFileContext(fileName, context) + + return fileName + +def taosdStart(): + cmd = "nohup /usr/bin/taosd 2>&1 & " + ret = exec(cmd) + print(f"exec taosd ret = {ret}\n") + time.sleep(3) + waitClusterAlive(10) + +def taosdStop(): + i = 1 + toBeKilled = "taosd" + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output(psCmd, shell=True) + while(processID): + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + print(f"i={i} kill taosd pid={processID}") + i += 1 + +def cleanAndStartTaosd(): + + # stop + taosdStop() + # clean + exec(f"rm -rf {dataDir}") + # start + taosdStart() + +def findContextValue(context, label): + start = context.find(label) + if start == -1 : + return "" + start += len(label) + 2 + # skip blank + while context[start] == ' ': + start += 1 + + # find end ',' + end = start + ends = [',','}',']', 0] + while context[end] not in ends: + end += 1 + + print(f"start = {start} end={end}\n") + return context[start:end] + + +def writeTemplateInfo(resultFile): + # create info + context = readFileContext(templateFile) + vgroups = findContextValue(context, "vgroups") + childCount = findContextValue(context, "childtable_count") + insertRows = findContextValue(context, "insert_rows") + line = f"vgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\n\n" + print(line) + appendFileContext(resultFile, line) + + +def totalCompressRate(algo, resultFile, writeSecond): + global Number + # flush + command = 'taos -s "flush database dbrate;"' + rets = exec(command) + command = 'taos -s "compact database dbrate;"' + rets = exec(command) + waitCompactFinish(60) + + # read compress rate + command = 'taos -s "show table distributed dbrate.meters\G;"' + rets = runRetList(command) + print(rets) + + str1 = rets[5] + arr = str1.split(" ") + + # Total_Size KB + str2 = arr[2] + pos = str2.find("=[") + totalSize = int(float(str2[pos+2:])/1024) + + # Compression_Ratio + str2 = arr[6] + pos = str2.find("=[") + rate = str2[pos+2:] + print("rate =" + rate) + + # total data file size + #dataSize = getFolderSize(f"{dataDir}/vnode/") + #dataSizeMB = int(dataSize/1024/1024) + + # appand to file + + Number += 1 + context = "%10s %10s %10s %10s %10s\n"%( Number, algo, str(totalSize)+" MB", rate+"%", writeSecond + " s") + showLog(context) + appendFileContext(resultFile, context) + + +def doTest(algo, resultFile): + print(f"doTest algo: {algo} \n") + #cleanAndStartTaosd() + + + # json + jsonFile = generateJsonFile(algo) + + # run taosBenchmark + t1 = time.time() + exec(f"taosBenchmark -f {jsonFile}") + t2 = time.time() + + # total compress rate + totalCompressRate(algo, resultFile, str(int(t2-t1))) + +def main(): + + # test compress method + algos = ["lz4", "zlib", "zstd", "xz", "disabled"] + + # record result + resultFile = "./result.txt" + timestamp = time.time() + now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) + context = f"\n---------------------- test rate ({now}) ---------------------------------\n" + + appendFileContext(resultFile, context) + # json info + writeTemplateInfo(resultFile) + # head + context = "\n%10s %10s %10s %10s %10s\n"%("No", "compress", "dataSize", "rate", "insertSeconds") + appendFileContext(resultFile, context) + + + # loop for all compression + for algo in algos: + # do test + doTest(algo, resultFile) + appendFileContext(resultFile, " \n") + + timestamp = time.time() + now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)) + appendFileContext(resultFile, f"\n{now} finished test!\n") + + +if __name__ == "__main__": + print("welcome use TDengine compress rate test tools.\n") + main() + # show result + print(resultContext) \ No newline at end of file