xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #28017 from taosdata/feat/TD-32250-3.0 

feat: add detect compression rate tools
This commit is contained in:
Alex Duan 2024-09-22 16:22:18 +08:00 committed by GitHub
parent 841dc81ea9 07bf98a595
commit 995f91e86e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 420 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
tools/auto/testCompression/json/d0_10w.csv.zip Normal file
View File

Binary file not shown.

64
tools/auto/testCompression/json/generate.json Normal file
View File

@ -0,0 +1,64 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"num_of_records_per_req": 500,
"thread_count": 4,
"create_table_thread_count": 1,
"confirm_parameter_prompt": "no",
"databases": [
{
"dbinfo": {
"name": "test",
"drop": "yes",
"vgroups": 2
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 1,
"insert_rows": 100000,
"childtable_prefix": "d",
"insert_mode": "taosc",
"timestamp_step": 1,
"start_timestamp":1700000000000,
"columns": [
{ "type": "bool", "name": "bc"},
{ "type": "float", "name": "fc", "max": 1, "min": 0 },
{ "type": "double", "name": "dc", "max": 1, "min": 0 },
{ "type": "tinyint", "name": "ti", "max": 100, "min": 0 },
{ "type": "smallint", "name": "si", "max": 100, "min": 0 },
{ "type": "int", "name": "ic", "max": 100, "min": 0 },
{ "type": "bigint", "name": "bi", "max": 100, "min": 0 },
{ "type": "utinyint", "name": "uti", "max": 100, "min": 0 },
{ "type": "usmallint", "name": "usi", "max": 100, "min": 0 },
{ "type": "uint", "name": "ui", "max": 100, "min": 0 },
{ "type": "ubigint", "name": "ubi", "max": 100, "min": 0 },
{ "type": "binary", "name": "bin", "len": 32},
{ "type": "nchar", "name": "nch", "len": 64}
],
"tags": [
{
"type": "tinyint",
"name": "groupid",
"max": 10,
"min": 1
},
{
"name": "location",
"type": "binary",
"len": 16,
"values": ["San Francisco", "Los Angles", "San Diego",
"San Jose", "Palo Alto", "Campbell", "Mountain View",
"Sunnyvale", "Santa Clara", "Cupertino"]
}
]
}
]
}
]
}

59
tools/auto/testCompression/json/template.json Normal file
View File

@ -0,0 +1,59 @@
{
"filetype": "insert",
"cfgdir": "/etc/taos",
"host": "127.0.0.1",
"port": 6030,
"user": "root",
"password": "taosdata",
"thread_count": 1,
"create_table_thread_count": 1,
"confirm_parameter_prompt": "no",
"num_of_records_per_req": 2000,
"prepared_rand": 100000,
"escape_character": "yes",
"databases": [
{
"dbinfo": {
"name": "dbrate",
"drop": "yes",
"vgroups": 1,
"stt_trigger": 1,
"wal_retention_size": 1,
"wal_retention_period": 1
},
"super_tables": [
{
"name": "meters",
"child_table_exists": "no",
"childtable_count": 1,
"childtable_prefix": "d",
"data_source": "sample",
"insert_mode": "taosc",
"insert_rows": 100000,
"timestamp_step": 1,
"start_timestamp": "2020-10-01 00:00:00.000",
"sample_file": "./json/d0_10w.csv",
"use_sample_ts": "yes",
"columns": [
{ "type": "bool", "name": "bc", "compress":"@COMPRESS"},
{ "type": "float", "name": "fc", "compress":"@COMPRESS"},
{ "type": "double", "name": "dc", "compress":"@COMPRESS"},
{ "type": "tinyint", "name": "ti", "compress":"@COMPRESS"},
{ "type": "smallint", "name": "si", "compress":"@COMPRESS"},
{ "type": "int", "name": "ic", "compress":"@COMPRESS"},
{ "type": "bigint", "name": "bi", "compress":"@COMPRESS"},
{ "type": "utinyint", "name": "uti","compress":"@COMPRESS"},
{ "type": "usmallint", "name": "usi","compress":"@COMPRESS"},
{ "type": "uint", "name": "ui", "compress":"@COMPRESS"},
{ "type": "ubigint", "name": "ubi","compress":"@COMPRESS"},
{ "type": "binary", "name": "bin", "len": 32, "compress":"@COMPRESS"},
{ "type": "nchar", "name": "nch", "len": 64, "compress":"@COMPRESS"}
],
"tags": [
{"type": "TINYINT", "name": "groupid", "max": 10, "min": 1}
]
}
]
}
]
}

297
tools/auto/testCompression/testCompression.py Normal file
View File

@ -0,0 +1,297 @@
import taos
import sys
import os
import subprocess
import time
import random
import json
from datetime import datetime
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
dataDir = "/var/lib/taos/"
templateFile = "json/template.json"
Number = 0
resultContext = ""
def showLog(str):
print(str)
def exec(command, show=True):
if(show):
print(f"exec {command}\n")
return os.system(command)
# run return output and error
def run(command, timeout = 60, show=True):
if(show):
print(f"run {command} timeout={timeout}s\n")
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process.wait(timeout)
output = process.stdout.read().decode(encoding="gbk")
error = process.stderr.read().decode(encoding="gbk")
return output, error
# return list after run
def runRetList(command, timeout=10):
output,error = run(command, timeout)
return output.splitlines()
def readFileContext(filename):
file = open(filename)
context = file.read()
file.close()
return context
def writeFileContext(filename, context):
file = open(filename, "w")
file.write(context)
file.close()
def appendFileContext(filename, context):
global resultContext
resultContext += context
try:
file = open(filename, "a")
wsize = file.write(context)
file.close()
except:
print(f"appand file error context={context} .")
def getFolderSize(folder):
total_size = 0
for dirpath, dirnames, filenames in os.walk(folder):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
total_size += os.path.getsize(filepath)
return total_size
def waitClusterAlive(loop):
for i in range(loop):
command = 'taos -s "show cluster alive\G;" '
out,err = run(command)
print(out)
if out.find("status: 1") >= 0:
showLog(f" i={i} wait cluster alive ok.\n")
return True
showLog(f" i={i} wait cluster alive ...\n")
time.sleep(1)
showLog(f" i={i} wait cluster alive failed.\n")
return False
def waitCompactFinish(loop):
for i in range(loop):
command = 'taos -s "show compacts;" '
out,err = run(command)
if out.find("Query OK, 0 row(s) in set") >= 0:
showLog(f" i={i} wait compact finish ok\n")
return True
showLog(f" i={i} wait compact ...\n")
time.sleep(1)
showLog(f" i={i} wait compact failed.\n")
return False
def getTypeName(datatype):
str1 = datatype.split(",")[0]
str2 = str1.split(":")[1]
str3 = str2.replace('"','').replace(' ','')
return str3
def getMatch(datatype, algo):
if algo == "tsz":
if datatype == "float" or datatype == "double":
return True
else:
return False
else:
return True
def generateJsonFile(algo):
print(f"doTest algo: {algo} \n")
# replace datatype
context = readFileContext(templateFile)
# replace compress
context = context.replace("@COMPRESS", algo)
# write to file
fileName = f"json/test_{algo}.json"
if os.path.exists(fileName):
os.remove(fileName)
writeFileContext(fileName, context)
return fileName
def taosdStart():
cmd = "nohup /usr/bin/taosd 2>&1 & "
ret = exec(cmd)
print(f"exec taosd ret = {ret}\n")
time.sleep(3)
waitClusterAlive(10)
def taosdStop():
i = 1
toBeKilled = "taosd"
killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -TERM > /dev/null 2>&1" % toBeKilled
psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled
processID = subprocess.check_output(psCmd, shell=True)
while(processID):
os.system(killCmd)
time.sleep(1)
processID = subprocess.check_output(psCmd, shell=True)
print(f"i={i} kill taosd pid={processID}")
i += 1
def cleanAndStartTaosd():
# stop
taosdStop()
# clean
exec(f"rm -rf {dataDir}")
# start
taosdStart()
def findContextValue(context, label):
start = context.find(label)
if start == -1 :
return ""
start += len(label) + 2
# skip blank
while context[start] == ' ':
start += 1
# find end ','
end = start
ends = [',','}',']', 0]
while context[end] not in ends:
end += 1
print(f"start = {start} end={end}\n")
return context[start:end]
def writeTemplateInfo(resultFile):
# create info
context = readFileContext(templateFile)
vgroups = findContextValue(context, "vgroups")
childCount = findContextValue(context, "childtable_count")
insertRows = findContextValue(context, "insert_rows")
line = f"vgroups = {vgroups}\nchildtable_count = {childCount}\ninsert_rows = {insertRows}\n\n"
print(line)
appendFileContext(resultFile, line)
def totalCompressRate(algo, resultFile, writeSecond):
global Number
# flush
command = 'taos -s "flush database dbrate;"'
rets = exec(command)
command = 'taos -s "compact database dbrate;"'
rets = exec(command)
waitCompactFinish(60)
# read compress rate
command = 'taos -s "show table distributed dbrate.meters\G;"'
rets = runRetList(command)
print(rets)
str1 = rets[5]
arr = str1.split(" ")
# Total_Size KB
str2 = arr[2]
pos = str2.find("=[")
totalSize = int(float(str2[pos+2:])/1024)
# Compression_Ratio
str2 = arr[6]
pos = str2.find("=[")
rate = str2[pos+2:]
print("rate =" + rate)
# total data file size
#dataSize = getFolderSize(f"{dataDir}/vnode/")
#dataSizeMB = int(dataSize/1024/1024)
# appand to file
Number += 1
context = "%10s %10s %10s %10s %10s\n"%( Number, algo, str(totalSize)+" MB", rate+"%", writeSecond + " s")
showLog(context)
appendFileContext(resultFile, context)
def doTest(algo, resultFile):
print(f"doTest algo: {algo} \n")
#cleanAndStartTaosd()
# json
jsonFile = generateJsonFile(algo)
# run taosBenchmark
t1 = time.time()
exec(f"taosBenchmark -f {jsonFile}")
t2 = time.time()
# total compress rate
totalCompressRate(algo, resultFile, str(int(t2-t1)))
def main():
# test compress method
algos = ["lz4", "zlib", "zstd", "xz", "disabled"]
# record result
resultFile = "./result.txt"
timestamp = time.time()
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
context = f"\n---------------------- test rate ({now}) ---------------------------------\n"
appendFileContext(resultFile, context)
# json info
writeTemplateInfo(resultFile)
# head
context = "\n%10s %10s %10s %10s %10s\n"%("No", "compress", "dataSize", "rate", "insertSeconds")
appendFileContext(resultFile, context)
# loop for all compression
for algo in algos:
# do test
doTest(algo, resultFile)
appendFileContext(resultFile, " \n")
timestamp = time.time()
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
appendFileContext(resultFile, f"\n{now} finished test!\n")
if __name__ == "__main__":
print("welcome use TDengine compress rate test tools.\n")
main()
# show result
print(resultContext)