Merge pull request #28017 from taosdata/feat/TD-32250-3.0
feat: add detect compression rate tools
This commit is contained in:
Alex Duan
GitHub
commit
995f91e86e
Binary file not shown.
|
|
@ -0,0 +1,64 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"num_of_records_per_req": 500,
|
||||
"thread_count": 4,
|
||||
"create_table_thread_count": 1,
|
||||
"confirm_parameter_prompt": "no",
|
||||
"databases": [
|
||||
{
|
||||
"dbinfo": {
|
||||
"name": "test",
|
||||
"drop": "yes",
|
||||
"vgroups": 2
|
||||
},
|
||||
"super_tables": [
|
||||
{
|
||||
"name": "meters",
|
||||
"child_table_exists": "no",
|
||||
"childtable_count": 1,
|
||||
"insert_rows": 100000,
|
||||
"childtable_prefix": "d",
|
||||
"insert_mode": "taosc",
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp":1700000000000,
|
||||
"columns": [
|
||||
{ "type": "bool", "name": "bc"},
|
||||
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
|
||||
{ "type": "double", "name": "dc", "max": 1, "min": 0 },
|
||||
{ "type": "tinyint", "name": "ti", "max": 100, "min": 0 },
|
||||
{ "type": "smallint", "name": "si", "max": 100, "min": 0 },
|
||||
{ "type": "int", "name": "ic", "max": 100, "min": 0 },
|
||||
{ "type": "bigint", "name": "bi", "max": 100, "min": 0 },
|
||||
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
|
||||
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
|
||||
{ "type": "uint", "name": "ui", "max": 100, "min": 0 },
|
||||
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0 },
|
||||
{ "type": "binary", "name": "bin", "len": 32},
|
||||
{ "type": "nchar", "name": "nch", "len": 64}
|
||||
],
|
||||
"tags": [
|
||||
{
|
||||
"type": "tinyint",
|
||||
"name": "groupid",
|
||||
"max": 10,
|
||||
"min": 1
|
||||
},
|
||||
{
|
||||
"name": "location",
|
||||
"type": "binary",
|
||||
"len": 16,
|
||||
"values": ["San Francisco", "Los Angles", "San Diego",
|
||||
"San Jose", "Palo Alto", "Campbell", "Mountain View",
|
||||
"Sunnyvale", "Santa Clara", "Cupertino"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
{
|
||||
"filetype": "insert",
|
||||
"cfgdir": "/etc/taos",
|
||||
"host": "127.0.0.1",
|
||||
"port": 6030,
|
||||
"user": "root",
|
||||
"password": "taosdata",
|
||||
"thread_count": 1,
|
||||
"create_table_thread_count": 1,
|
||||
"confirm_parameter_prompt": "no",
|
||||
"num_of_records_per_req": 2000,
|
||||
"prepared_rand": 100000,
|
||||
"escape_character": "yes",
|
||||
"databases": [
|
||||
{
|
||||
"dbinfo": {
|
||||
"name": "dbrate",
|
||||
"drop": "yes",
|
||||
"vgroups": 1,
|
||||
"stt_trigger": 1,
|
||||
"wal_retention_size": 1,
|
||||
"wal_retention_period": 1
|
||||
},
|
||||
"super_tables": [
|
||||
{
|
||||
"name": "meters",
|
||||
"child_table_exists": "no",
|
||||
"childtable_count": 1,
|
||||
"childtable_prefix": "d",
|
||||
"data_source": "sample",
|
||||
"insert_mode": "taosc",
|
||||
"insert_rows": 100000,
|
||||
"timestamp_step": 1,
|
||||
"start_timestamp": "2020-10-01 00:00:00.000",
|
||||
"sample_file": "./json/d0_10w.csv",
|
||||
"use_sample_ts": "yes",
|
||||
"columns": [
|
||||
{ "type": "bool", "name": "bc", "compress":"@COMPRESS"},
|
||||
{ "type": "float", "name": "fc", "compress":"@COMPRESS"},
|
||||
{ "type": "double", "name": "dc", "compress":"@COMPRESS"},
|
||||
{ "type": "tinyint", "name": "ti", "compress":"@COMPRESS"},
|
||||
{ "type": "smallint", "name": "si", "compress":"@COMPRESS"},
|
||||
{ "type": "int", "name": "ic", "compress":"@COMPRESS"},
|
||||
{ "type": "bigint", "name": "bi", "compress":"@COMPRESS"},
|
||||
{ "type": "utinyint", "name": "uti","compress":"@COMPRESS"},
|
||||
{ "type": "usmallint", "name": "usi","compress":"@COMPRESS"},
|
||||
{ "type": "uint", "name": "ui", "compress":"@COMPRESS"},
|
||||
{ "type": "ubigint", "name": "ubi","compress":"@COMPRESS"},
|
||||
{ "type": "binary", "name": "bin", "len": 32, "compress":"@COMPRESS"},
|
||||
{ "type": "nchar", "name": "nch", "len": 64, "compress":"@COMPRESS"}
|
||||
],
|
||||
"tags": [
|
||||
{"type": "TINYINT", "name": "groupid", "max": 10, "min": 1}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,297 @@
|
|||
import taos
|
||||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
###################################################################
|
||||
# Copyright (c) 2016 by TAOS Technologies, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This file is proprietary and confidential to TAOS Technologies.
|
||||
# No part of this file may be reproduced, stored, transmitted,
|
||||
# disclosed or used in any form or by any means other than as
|
||||
# expressly provided by the written permission from Jianhui Tao
|
||||
#
|
||||
###################################################################
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
dataDir = "/var/lib/taos/"
|
||||
templateFile = "json/template.json"
|
||||
Number = 0
|
||||
resultContext = ""
|
||||
|
||||
|
||||
def showLog(str):
|
||||
print(str)
|
||||
|
||||
def exec(command, show=True):
|
||||
if(show):
|
||||
print(f"exec {command}\n")
|
||||
return os.system(command)
|
||||
|
||||
# run return output and error
|
||||
def run(command, timeout = 60, show=True):
|
||||
if(show):
|
||||
print(f"run {command} timeout={timeout}s\n")
|
||||
|
||||
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
process.wait(timeout)
|
||||
|
||||
output = process.stdout.read().decode(encoding="gbk")
|
||||
error = process.stderr.read().decode(encoding="gbk")
|
||||
|
||||
return output, error
|
||||
|
||||
# return list after run
|
||||
def runRetList(command, timeout=10):
|
||||
output,error = run(command, timeout)
|
||||
return output.splitlines()
|
||||
|
||||
def readFileContext(filename):
|
||||
file = open(filename)
|
||||
context = file.read()
|
||||
file.close()
|
||||
return context
|
||||
|
||||
|
||||
def writeFileContext(filename, context):
|
||||
file = open(filename, "w")
|
||||
file.write(context)
|
||||
file.close()
|
||||
|
||||
def appendFileContext(filename, context):
|
||||
global resultContext
|
||||
resultContext += context
|
||||
try:
|
||||
file = open(filename, "a")
|
||||
wsize = file.write(context)
|
||||
file.close()
|
||||
except:
|
||||
print(f"appand file error context={context} .")
|
||||
|
||||
def getFolderSize(folder):
|
||||
total_size = 0
|
||||
for dirpath, dirnames, filenames in os.walk(folder):
|
||||
for filename in filenames:
|
||||
filepath = os.path.join(dirpath, filename)
|
||||
total_size += os.path.getsize(filepath)
|
||||
return total_size
|
||||
|
||||
def waitClusterAlive(loop):
|
||||
for i in range(loop):
|
||||
command = 'taos -s "show cluster alive\G;" '
|
||||
out,err = run(command)
|
||||
print(out)
|
||||
if out.find("status: 1") >= 0:
|
||||
showLog(f" i={i} wait cluster alive ok.\n")
|
||||
return True
|
||||
|
||||
showLog(f" i={i} wait cluster alive ...\n")
|
||||
time.sleep(1)
|
||||
|
||||
showLog(f" i={i} wait cluster alive failed.\n")
|
||||
return False
|
||||
|
||||
def waitCompactFinish(loop):
|
||||
for i in range(loop):
|
||||
command = 'taos -s "show compacts;" '
|
||||
out,err = run(command)
|
||||
if out.find("Query OK, 0 row(s) in set") >= 0:
|
||||
showLog(f" i={i} wait compact finish ok\n")
|
||||
return True
|
||||
|
||||
showLog(f" i={i} wait compact ...\n")
|
||||
time.sleep(1)
|
||||
|
||||
showLog(f" i={i} wait compact failed.\n")
|
||||
return False
|
||||
|
||||
|
||||
def getTypeName(datatype):
|
||||
str1 = datatype.split(",")[0]
|
||||
str2 = str1.split(":")[1]
|
||||
str3 = str2.replace('"','').replace(' ','')
|
||||
return str3
|
||||
|
||||
|
||||
def getMatch(datatype, algo):
|
||||
if algo == "tsz":
|
||||
if datatype == "float" or datatype == "double":
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def generateJsonFile(algo):
|
||||
print(f"doTest algo: {algo} \n")
|
||||
|
||||
# replace datatype
|
||||
context = readFileContext(templateFile)
|
||||
# replace compress
|
||||
context = context.replace("@COMPRESS", algo)
|
||||
|
||||
# write to file
|
||||
fileName = f"json/test_{algo}.json"
|
||||
if os.path.exists(fileName):
|
||||
os.remove(fileName)
|
||||
writeFileContext(fileName, context)
|
||||
|
||||
return fileName
|
||||
|
||||
def taosdStart():
|
||||
cmd = "nohup /usr/bin/taosd 2>&1 & "
|
||||
ret = exec(cmd)
|
||||
print(f"exec taosd ret = {ret}\n")
|
||||
time.sleep(3)
|
||||
waitClusterAlive(10)
|
||||
|
||||
def taosdStop():
|
||||
i = 1
|
||||
toBeKilled = "taosd"
|
||||
killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled
|
||||
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
|
||||
processID = subprocess.check_output(psCmd, shell=True)
|
||||
while(processID):
|
||||
os.system(killCmd)
|
||||
time.sleep(1)
|
||||
processID = subprocess.check_output(psCmd, shell=True)
|
||||
print(f"i={i} kill taosd pid={processID}")
|
||||
i += 1
|
||||
|
||||
def cleanAndStartTaosd():
|
||||
|
||||
# stop
|
||||
taosdStop()
|
||||
# clean
|
||||
exec(f"rm -rf {dataDir}")
|
||||
# start
|
||||
taosdStart()
|
||||
|
||||
def findContextValue(context, label):
|
||||
start = context.find(label)
|
||||
if start == -1 :
|
||||
return ""
|
||||
start += len(label) + 2
|
||||
# skip blank
|
||||
while context[start] == ' ':
|
||||
start += 1
|
||||
|
||||
# find end ','
|
||||
end = start
|
||||
ends = [',','}',']', 0]
|
||||
while context[end] not in ends:
|
||||
end += 1
|
||||
|
||||
print(f"start = {start} end={end}\n")
|
||||
return context[start:end]
|
||||
|
||||
|
||||
def writeTemplateInfo(resultFile):
|
||||
# create info
|
||||
context = readFileContext(templateFile)
|
||||
vgroups = findContextValue(context, "vgroups")
|
||||
childCount = findContextValue(context, "childtable_count")
|
||||
insertRows = findContextValue(context, "insert_rows")
|
||||
line = f"vgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\n\n"
|
||||
print(line)
|
||||
appendFileContext(resultFile, line)
|
||||
|
||||
|
||||
def totalCompressRate(algo, resultFile, writeSecond):
|
||||
global Number
|
||||
# flush
|
||||
command = 'taos -s "flush database dbrate;"'
|
||||
rets = exec(command)
|
||||
command = 'taos -s "compact database dbrate;"'
|
||||
rets = exec(command)
|
||||
waitCompactFinish(60)
|
||||
|
||||
# read compress rate
|
||||
command = 'taos -s "show table distributed dbrate.meters\G;"'
|
||||
rets = runRetList(command)
|
||||
print(rets)
|
||||
|
||||
str1 = rets[5]
|
||||
arr = str1.split(" ")
|
||||
|
||||
# Total_Size KB
|
||||
str2 = arr[2]
|
||||
pos = str2.find("=[")
|
||||
totalSize = int(float(str2[pos+2:])/1024)
|
||||
|
||||
# Compression_Ratio
|
||||
str2 = arr[6]
|
||||
pos = str2.find("=[")
|
||||
rate = str2[pos+2:]
|
||||
print("rate =" + rate)
|
||||
|
||||
# total data file size
|
||||
#dataSize = getFolderSize(f"{dataDir}/vnode/")
|
||||
#dataSizeMB = int(dataSize/1024/1024)
|
||||
|
||||
# appand to file
|
||||
|
||||
Number += 1
|
||||
context = "%10s %10s %10s %10s %10s\n"%( Number, algo, str(totalSize)+" MB", rate+"%", writeSecond + " s")
|
||||
showLog(context)
|
||||
appendFileContext(resultFile, context)
|
||||
|
||||
|
||||
def doTest(algo, resultFile):
|
||||
print(f"doTest algo: {algo} \n")
|
||||
#cleanAndStartTaosd()
|
||||
|
||||
|
||||
# json
|
||||
jsonFile = generateJsonFile(algo)
|
||||
|
||||
# run taosBenchmark
|
||||
t1 = time.time()
|
||||
exec(f"taosBenchmark -f {jsonFile}")
|
||||
t2 = time.time()
|
||||
|
||||
# total compress rate
|
||||
totalCompressRate(algo, resultFile, str(int(t2-t1)))
|
||||
|
||||
def main():
|
||||
|
||||
# test compress method
|
||||
algos = ["lz4", "zlib", "zstd", "xz", "disabled"]
|
||||
|
||||
# record result
|
||||
resultFile = "./result.txt"
|
||||
timestamp = time.time()
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
|
||||
context = f"\n---------------------- test rate ({now}) ---------------------------------\n"
|
||||
|
||||
appendFileContext(resultFile, context)
|
||||
# json info
|
||||
writeTemplateInfo(resultFile)
|
||||
# head
|
||||
context = "\n%10s %10s %10s %10s %10s\n"%("No", "compress", "dataSize", "rate", "insertSeconds")
|
||||
appendFileContext(resultFile, context)
|
||||
|
||||
|
||||
# loop for all compression
|
||||
for algo in algos:
|
||||
# do test
|
||||
doTest(algo, resultFile)
|
||||
appendFileContext(resultFile, " \n")
|
||||
|
||||
timestamp = time.time()
|
||||
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
|
||||
appendFileContext(resultFile, f"\n{now} finished test!\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("welcome use TDengine compress rate test tools.\n")
|
||||
main()
|
||||
# show result
|
||||
print(resultContext)
|
||||
Loading…
Reference in New Issue