Merge pull request #27762 from taosdata/fix/TS-5035-3.0
feat: auto test all datatype compression rate tools
This commit is contained in:
Alex Duan
GitHub
commit
4d11f31955
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 4,
|
||||
"create_table_thread_count": 1,
|
||||
"confirm_parameter_prompt": "no",
|
||||
"num_of_records_per_req": 1000,
|
||||
"prepared_rand": 10000,
|
||||
"escape_character": "yes",
|
||||
"databases": [
|
||||
{
|
||||
"dbinfo": {
|
||||
"name": "dbrate",
|
||||
"drop": "yes",
|
||||
"vgroups": 4,
|
||||
"stt_trigger": 1,
|
||||
"wal_retention_size": 1,
|
||||
"wal_retention_period": 1
|
||||
},
|
||||
"super_tables": [
|
||||
{
|
||||
"name": "meters",
|
||||
"child_table_exists": "no",
|
||||
"childtable_count": 10000,
|
||||
"childtable_prefix": "d",
|
||||
"data_source": "rand",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 100,
|
||||
"timestamp_step": 1000,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"columns": [
|
||||
@DATATYPE
|
||||
],
|
||||
"tags": [
|
||||
{"type": "TINYINT", "name": "groupid", "max": 10, "min": 1}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
import taos
|
||||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
dataDir = "/var/lib/taos"
|
||||
templateFile = "json/template.json"
|
||||
Number = 0
|
||||
|
||||
def showLog(str):
|
||||
print(str)
|
||||
|
||||
def exec(command, show=True):
|
||||
if(show):
|
||||
print(f"exec {command}\n")
|
||||
return os.system(command)
|
||||
|
||||
# run return output and error
|
||||
def run(command, timeout = 60, show=True):
|
||||
if(show):
|
||||
print(f"run {command} timeout={timeout}s\n")
|
||||
|
||||
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
process.wait(timeout)
|
||||
|
||||
output = process.stdout.read().decode(encoding="gbk")
|
||||
error = process.stderr.read().decode(encoding="gbk")
|
||||
|
||||
return output, error
|
||||
|
||||
# return list after run
|
||||
def runRetList(command, timeout=10):
|
||||
output,error = run(command, timeout)
|
||||
return output.splitlines()
|
||||
|
||||
def readFileContext(filename):
|
||||
file = open(filename)
|
||||
context = file.read()
|
||||
file.close()
|
||||
return context
|
||||
|
||||
|
||||
def writeFileContext(filename, context):
|
||||
file = open(filename, "w")
|
||||
file.write(context)
|
||||
file.close()
|
||||
|
||||
def appendFileContext(filename, context):
|
||||
try:
|
||||
file = open(filename, "a")
|
||||
wsize = file.write(context)
|
||||
file.close()
|
||||
except:
|
||||
print(f"appand file error context={context} .")
|
||||
|
||||
def getFolderSize(folder):
|
||||
total_size = 0
|
||||
for dirpath, dirnames, filenames in os.walk(folder):
|
||||
for filename in filenames:
|
||||
filepath = os.path.join(dirpath, filename)
|
||||
total_size += os.path.getsize(filepath)
|
||||
return total_size
|
||||
|
||||
def waitClusterAlive(loop):
|
||||
for i in range(loop):
|
||||
command = 'taos -s "show cluster alive\G;" '
|
||||
out,err = run(command)
|
||||
print(out)
|
||||
if out.find("status: 1") >= 0:
|
||||
showLog(f" i={i} wait cluster alive ok.\n")
|
||||
return True
|
||||
|
||||
showLog(f" i={i} wait cluster alive ...\n")
|
||||
time.sleep(1)
|
||||
|
||||
showLog(f" i={i} wait cluster alive failed.\n")
|
||||
return False
|
||||
|
||||
def waitCompactFinish(loop):
|
||||
for i in range(loop):
|
||||
command = 'taos -s "show compacts;" '
|
||||
out,err = run(command)
|
||||
if out.find("Query OK, 0 row(s) in set") >= 0:
|
||||
showLog(f" i={i} wait compact finish ok\n")
|
||||
return True
|
||||
|
||||
showLog(f" i={i} wait compact ...\n")
|
||||
time.sleep(1)
|
||||
|
||||
showLog(f" i={i} wait compact failed.\n")
|
||||
return False
|
||||
|
||||
|
||||
def getTypeName(datatype):
|
||||
str1 = datatype.split(",")[0]
|
||||
str2 = str1.split(":")[1]
|
||||
str3 = str2.replace('"','').replace(' ','')
|
||||
return str3
|
||||
|
||||
|
||||
def getMatch(datatype, algo):
|
||||
if algo == "tsz":
|
||||
if datatype == "float" or datatype == "double":
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def generateJsonFile(datatype, typeName, algo):
|
||||
print(f"doTest dataType: {typeName} algo: {algo} \n")
|
||||
|
||||
# replace compress
|
||||
datatype = datatype.replace("@COMPRESS", algo)
|
||||
# replace datatype
|
||||
context = readFileContext(templateFile)
|
||||
context = context.replace("@DATATYPE", datatype)
|
||||
# write to file
|
||||
fileName = f"json/{typeName}.json"
|
||||
if os.path.exists(fileName):
|
||||
os.remove(fileName)
|
||||
writeFileContext(fileName, context)
|
||||
|
||||
return fileName
|
||||
|
||||
def taosdStart():
|
||||
cmd = "nohup /usr/bin/taosd 2>&1 & "
|
||||
ret = exec(cmd)
|
||||
print(f"exec taosd ret = {ret}\n")
|
||||
time.sleep(3)
|
||||
waitClusterAlive(10)
|
||||
|
||||
def taosdStop():
|
||||
i = 1
|
||||
toBeKilled = "taosd"
|
||||
killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled
|
||||
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
|
||||
processID = subprocess.check_output(psCmd, shell=True)
|
||||
while(processID):
|
||||
os.system(killCmd)
|
||||
time.sleep(1)
|
||||
processID = subprocess.check_output(psCmd, shell=True)
|
||||
print(f"i={i} kill taosd pid={processID}")
|
||||
i += 1
|
||||
|
||||
def cleanAndStartTaosd():
|
||||
|
||||
taosdStop()
|
||||
# clear
|
||||
exec(f"rm -rf {dataDir}")
|
||||
# start
|
||||
taosdStart()
|
||||
|
||||
def findContextValue(context, label):
|
||||
start = context.find(label)
|
||||
if start == -1 :
|
||||
return ""
|
||||
start += len(label) + 2
|
||||
# skip blank
|
||||
while context[start] == ' ':
|
||||
start += 1
|
||||
|
||||
# find end ','
|
||||
end = start
|
||||
ends = [',','}',']', 0]
|
||||
while context[end] not in ends:
|
||||
end += 1
|
||||
|
||||
print(f"start = {start} end={end}\n")
|
||||
return context[start:end]
|
||||
|
||||
|
||||
def writeTemplateInfo(resultFile):
|
||||
context = readFileContext(templateFile)
|
||||
vgroups = findContextValue(context, "vgroups")
|
||||
childCount = findContextValue(context, "childtable_count")
|
||||
insertRows = findContextValue(context, "insert_rows")
|
||||
line = f"vgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\n\n"
|
||||
print(line)
|
||||
appendFileContext(resultFile, line)
|
||||
|
||||
def totalCompressRate(typeName, algo, resultFile):
|
||||
global Number
|
||||
# flush
|
||||
command = 'taos -s "flush database dbrate;"'
|
||||
rets = exec(command)
|
||||
command = 'taos -s "compact database dbrate;"'
|
||||
rets = exec(command)
|
||||
waitCompactFinish(60)
|
||||
|
||||
# read compress rate
|
||||
command = 'taos -s "show table distributed dbrate.meters\G;"'
|
||||
rets = runRetList(command)
|
||||
print(rets)
|
||||
str1 = rets[5]
|
||||
str2 = str1.split(" ")[6]
|
||||
pos = str2.find("=[")
|
||||
rate = str2[pos+2:]
|
||||
print("rate =" + rate)
|
||||
|
||||
# total data file size
|
||||
dataSize = getFolderSize(f"{dataDir}/vnode/")
|
||||
dataSizeMB = int(dataSize/1024/1024)
|
||||
|
||||
# appand to file
|
||||
|
||||
Number += 1
|
||||
context = "%10s %10s %10s %10s %10s\n"%( Number, algo, typeName, str(dataSizeMB)+"M", rate+"%")
|
||||
showLog(context)
|
||||
appendFileContext(resultFile, context)
|
||||
|
||||
|
||||
def doTest(dataType, typeName, algo, resultFile):
|
||||
print(f"doTest dataType: {typeName} algo: {algo} \n")
|
||||
cleanAndStartTaosd()
|
||||
|
||||
# json
|
||||
jsonFile = generateJsonFile(dataType, typeName, algo)
|
||||
# run taosBenchmark
|
||||
exec(f"taosBenchmark -f {jsonFile}")
|
||||
|
||||
# total compress rate
|
||||
totalCompressRate(typeName, algo, resultFile)
|
||||
|
||||
def main():
|
||||
# test types
|
||||
dataTypes = [
|
||||
'{"type": "bool", "count":100, "min": 0, "max": 1, "compress":"@COMPRESS"}',
|
||||
'{"type": "tinyint", "count":100, "min": 0, "max": 100, "compress":"@COMPRESS"}',
|
||||
'{"type": "smallint", "count":100, "min": 0, "max": 1000, "compress":"@COMPRESS"}',
|
||||
'{"type": "int", "count":100, "min": 0, "max": 100000, "compress":"@COMPRESS"}',
|
||||
'{"type": "bigint", "count":100, "min": 0, "max": 1000000, "compress":"@COMPRESS"}',
|
||||
'{"type": "float", "count":100, "min": 0, "max": 1000, "compress":"@COMPRESS"}',
|
||||
'{"type": "double", "count":100, "min": 0, "max": 1000, "compress":"@COMPRESS"}',
|
||||
'{"type": "binary", "count":100, "len": 64, "compress":"@COMPRESS"}',
|
||||
'{"type": "nchar", "count":100, "len": 64, "compress":"@COMPRESS"}'
|
||||
]
|
||||
|
||||
# test compress method
|
||||
algos = ["lz4", "zlib", "zstd", "xz", "tsz"]
|
||||
|
||||
# record result
|
||||
resultFile = "./result.txt"
|
||||
timestamp = time.time()
|
||||
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
|
||||
context = f"\n---------------------- test rate ({now}) ---------------------------------\n"
|
||||
appendFileContext(resultFile, context)
|
||||
# json info
|
||||
writeTemplateInfo(resultFile)
|
||||
# head
|
||||
context = "%10s %10s %10s %10s %10s\n"%("No","compress","dataType", "dataSize","rate")
|
||||
appendFileContext(resultFile, context)
|
||||
|
||||
|
||||
# loop for all types
|
||||
for dataType in dataTypes:
|
||||
typeName = getTypeName(dataType)
|
||||
for algo in algos:
|
||||
if getMatch(typeName, algo) is False:
|
||||
print(f"warning dataType {typeName} not match with algo {algo} \n")
|
||||
continue
|
||||
# do test
|
||||
doTest(dataType, typeName, algo, resultFile)
|
||||
appendFileContext(resultFile, " \n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("welcome use TDengine compress rate test tools.\n")
|
||||
main()
|
||||
Loading…
Reference in New Issue