xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

update script to support wal param

This commit is contained in:
Shuduo Sang 2020-06-08 11:16:13 +08:00
parent 63bf9970aa
commit fb1f837d28
8 changed files with 124 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/perftest-scripts/perftest-csv2png.gnuplot
View File

@ -2,7 +2,7 @@
reset reset
set terminal png set terminal png
set title "Performance Test Report" font ",20" set title filename font ",20"
set ylabel "Time in Seconds" set ylabel "Time in Seconds"

74
tests/perftest-scripts/perftest-daily.sh
View File

@ -16,9 +16,26 @@ function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") &&
function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } #
function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } #
function set-Wal { function setMaxConnections {
echo "/etc/taos/taos.cfg maxConnection will be set to $1"
hasText=`grep "maxConnections" /etc/taos/taos.cfg`
if [[ -z "$hasText" ]]; then
echo "maxConnections $1" >> /etc/taos/taos.cfg
else
sed -i 's/^maxConnections.*$/maxConnections '"$1"'/g' /etc/taos/taos.cfg
fi
}
function setWal {
echo "/etc/taos/taos.cfg walLevel will be set to $1" echo "/etc/taos/taos.cfg walLevel will be set to $1"
sed -i 's/^walLevel.*$/walLevel '"$1"'/g' /etc/taos/taos.cfg
hasText=`grep "walLevel" /etc/taos/taos.cfg`
if [[ -z "$hasText" ]]; then
echo "walLevel $1" >> /etc/taos/taos.cfg
else
sed -i 's/^walLevel.*$/walLevel '"$1"'/g' /etc/taos/taos.cfg
fi
} }
function collectSysInfo { function collectSysInfo {
@ -70,15 +87,25 @@ function sendReport {
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
echo -e "to: ${receiver}\nsubject: Perf test report ${today}, commit ID: ${LOCAL_COMMIT}\n" | \ echo -e "to: ${receiver}\nsubject: Perf test report ${today}, commit ID: ${LOCAL_COMMIT}\n" | \
(cat - && uuencode perftest-1d-$today.log perftest-1d-$today.log)| \ (cat - && uuencode perftest-1d-wal1-$today.log perftest-1d-wal1-$today.log)| \
(cat - && uuencode perftest-1d-report.csv perftest-1d-report-$today.csv) | \ (cat - && uuencode perftest-1d-wal1-report.csv perftest-1d-wal1-report-$today.csv) | \
(cat - && uuencode perftest-1d-report.png perftest-1d-report-$today.png) | \ (cat - && uuencode perftest-1d-wal1-report.png perftest-1d-wal1-report-$today.png) | \
(cat - && uuencode perftest-13d-$today.log perftest-13d-$today.log)| \ (cat - && uuencode perftest-13d-wal1-$today.log perftest-13d-wal1-$today.log)| \
(cat - && uuencode perftest-13d-report.csv perftest-13d-report-$today.csv) | \ (cat - && uuencode perftest-13d-wal1-report.csv perftest-13d-wal1-report-$today.csv) | \
(cat - && uuencode perftest-13d-report.png perftest-13d-report-$today.png) | \ (cat - && uuencode perftest-13d-wal1-report.png perftest-13d-wal1-report-$today.png) | \
(cat - && uuencode taosdemo-$today.log taosdemo-$today.log) | \ (cat - && uuencode taosdemo-wal1-$today.log taosdemo-wal1-$today.log) | \
(cat - && uuencode taosdemo-report.csv taosdemo-report-$today.csv) | \ (cat - && uuencode taosdemo-wal1-report.csv taosdemo-wal1-report-$today.csv) | \
(cat - && uuencode taosdemo-report.png taosdemo-report-$today.png) | \ (cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \
(cat - && uuencode taosdemo-wal1-report.png taosdemo-wal1-report-$today.png) | \
(cat - && uuencode perftest-1d-wal2-$today.log perftest-1d-wal2-$today.log)| \
(cat - && uuencode perftest-1d-wal2-report.csv perftest-1d-wal2-report-$today.csv) | \
(cat - && uuencode perftest-1d-wal2-report.png perftest-1d-wal2-report-$today.png) | \
(cat - && uuencode perftest-13d-wal2-$today.log perftest-13d-wal2-$today.log)| \
(cat - && uuencode perftest-13d-wal2-report.csv perftest-13d-wal2-report-$today.csv) | \
(cat - && uuencode perftest-13d-wal2-report.png perftest-13d-wal2-report-$today.png) | \
(cat - && uuencode taosdemo-wal2-$today.log taosdemo-wal2-$today.log) | \
(cat - && uuencode taosdemo-wal2-report.csv taosdemo-wal2-report-$today.csv) | \
(cat - && uuencode taosdemo-rps-wal2-report.csv taosdemo-rps-wal2-report-$today.csv) | \
(cat - && uuencode sysinfo.log sysinfo.txt) | \ (cat - && uuencode sysinfo.log sysinfo.txt) | \
(cat - && uuencode taos.cfg taos-cfg-$today.txt) | \ (cat - && uuencode taos.cfg taos-cfg-$today.txt) | \
ssmtp "${receiver}" ssmtp "${receiver}"
@ -91,17 +118,34 @@ echo -e "cron-ran-at-${today}" >> cron.log
echoInfo "Build TDengine" echoInfo "Build TDengine"
buildTDengine buildTDengine
set-Wal "2" ############################
setMaxConnections 100
############################
setWal "2"
cd /root cd /root
./perftest-tsdb-compare-1d.sh ./perftest-tsdb-compare-1d.sh "wal2"
cd /root cd /root
./perftest-tsdb-compare-13d.sh ./perftest-tsdb-compare-13d.sh "wal2"
cd /root cd /root
./perftest-taosdemo.sh ./perftest-taosdemo.sh "wal2"
#############################
setWal "1"
cd /root
./perftest-tsdb-compare-1d.sh "wal1"
cd /root
./perftest-tsdb-compare-13d.sh "wal1"
cd /root
./perftest-taosdemo.sh "wal1"
#############################
collectSysInfo collectSysInfo
echoInfo "Send Report" echoInfo "Send Report"

72
tests/perftest-scripts/perftest-taosdemo.sh
View File

@ -1,20 +1,20 @@
#!/bin/bash #!/bin/bash
# Coloured Echoes # # Coloured Echoes
function red_echo { echo -e "\033[31m$@\033[0m"; } # function red_echo { echo -e "\033[31m$@\033[0m"; }
function green_echo { echo -e "\033[32m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; }
function yellow_echo { echo -e "\033[33m$@\033[0m"; } # function yellow_echo { echo -e "\033[33m$@\033[0m"; }
function white_echo { echo -e "\033[1;37m$@\033[0m"; } # function white_echo { echo -e "\033[1;37m$@\033[0m"; }
# Coloured Printfs # # Coloured Printfs
function red_printf { printf "\033[31m$@\033[0m"; } # function red_printf { printf "\033[31m$@\033[0m"; }
function green_printf { printf "\033[32m$@\033[0m"; } # function green_printf { printf "\033[32m$@\033[0m"; }
function yellow_printf { printf "\033[33m$@\033[0m"; } # function yellow_printf { printf "\033[33m$@\033[0m"; }
function white_printf { printf "\033[1;37m$@\033[0m"; } # function white_printf { printf "\033[1;37m$@\033[0m"; }
# Debugging Outputs # # Debugging Outputs
function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; }
function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; }
function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; }
function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; }
function restartTaosd { function restartTaosd {
systemctl stop taosd systemctl stop taosd
@ -32,39 +32,57 @@ function runCreateTableOnly {
echoInfo "Restart Taosd" echoInfo "Restart Taosd"
restartTaosd restartTaosd
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$today.log" /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$1-$today.log"
demoTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'` demoCreateTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'`
}
function runDeleteTableOnly {
echoInfo "Restart Taosd"
restartTaosd
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$1-$today.log"
demoDeleteTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'`
} }
function runCreateTableThenInsert { function runCreateTableThenInsert {
echoInfo "Restart Taosd" echoInfo "Restart Taosd"
restartTaosd restartTaosd
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$today.log" /usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$1-$today.log"
demoTableAndInsert=`grep "Total:" totaltime.out|awk '{print $2}'` demoTableAndInsert=`grep "Total:" totaltime.out|awk '{print $2}'`
demoRPS=`grep "records\/second" taosdemo-$today.log | tail -n1 | awk '{print $13}'` demoRPS=`grep "records\/second" taosdemo-$1-$today.log | tail -n1 | awk '{print $13}'`
} }
function generateTaosdemoPlot { function generateTaosdemoPlot {
echo "${today}, demoTableOnly: ${demoTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log echo "${today} $1, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log
echo "${today}, ${demoTableOnly}, ${demoTableAndInsert}, ${demoRPS}" >> taosdemo-report.csv echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$1-report.csv
echo "${today}, ${demoRPS}" >> taosdemo-rps-$1-report.csv
csvLines=`cat taosdemo-report.csv | wc -l` csvLines=`cat taosdemo-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then if [ "$csvLines" -gt "10" ]; then
sed -i '1d' taosdemo-report.csv sed -i '1d' taosdemo-$1-report.csv
fi fi
gnuplot -p taosdemo-csv2png.gnuplot csvLines=`cat taosdemo-rps-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then
sed -i '1d' taosdemo-rps-$1-report.csv
fi
gnuplot -e "filename='taosdemo-$1-report'" -p taosdemo-csv2png.gnuplot
gnuplot -e "filename='taosdemo-rps-$1-report'" -p taosdemo-rps-csv2png.gnuplot
} }
today=`date +"%Y%m%d"` today=`date +"%Y%m%d"`
cd /root cd /root
echoInfo "Test Create Table Only " echoInfo "Test Create Table Only "
runCreateTableOnly runCreateTableOnly $1
echoInfo "Test Create Table then Insert data" echoInfo "Test Create Table then Insert data"
runCreateTableThenInsert runDeleteTableOnly $1
echoInfo "Test Create Table then Insert data"
runCreateTableThenInsert $1
echoInfo "Generate plot for taosdemo" echoInfo "Generate plot for taosdemo"
generateTaosdemoPlot generateTaosdemoPlot $1
echoInfo "End of TaosDemo Test" echoInfo "End of TaosDemo Test"

12
tests/perftest-scripts/perftest-tsdb-compare-13d.sh
View File

@ -33,26 +33,26 @@ function runPerfTest13d {
restartTaosd restartTaosd
cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare
./runreal-13d-csv.sh 2>&1 | tee /root/perftest-13d-$today.log ./runreal-13d-csv.sh $1 2>&1 | tee /root/perftest-13d-$1-$today.log
} }
function generatePerfPlot13d { function generatePerfPlot13d {
cd /root cd /root
csvLines=`cat perftest-13d-report.csv | wc -l` csvLines=`cat perftest-13d-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then if [ "$csvLines" -gt "10" ]; then
sed -i '1d' perftest-13d-report.csv sed -i '1d' perftest-13d-$1-report.csv
fi fi
gnuplot -e "filename='perftest-13d-report'" -p perftest-csv2png.gnuplot gnuplot -e "filename='perftest-13d-$1-report'" -p perftest-csv2png.gnuplot
} }
today=`date +"%Y%m%d"` today=`date +"%Y%m%d"`
cd /root cd /root
echoInfo "run Performance Test with 13 days data" echoInfo "run Performance Test with 13 days data"
runPerfTest13d runPerfTest13d $1
echoInfo "Generate plot of 13 days data" echoInfo "Generate plot of 13 days data"
generatePerfPlot13d generatePerfPlot13d $1
echoInfo "End of TSDB-Compare 13-days-data Test" echoInfo "End of TSDB-Compare 13-days-data Test"

12
tests/perftest-scripts/perftest-tsdb-compare-1d.sh
View File

@ -33,26 +33,26 @@ function runPerfTest1d {
restartTaosd restartTaosd
cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare
./runreal-1d-csv.sh 2>&1 | tee /root/perftest-1d-$today.log ./runreal-1d-csv.sh $1 2>&1 | tee /root/perftest-1d-$1-$today.log
} }
function generatePerfPlot1d { function generatePerfPlot1d {
cd /root cd /root
csvLines=`cat perftest-1d-report.csv | wc -l` csvLines=`cat perftest-1d-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then if [ "$csvLines" -gt "10" ]; then
sed -i '2d' perftest-1d-report.csv sed -i '2d' perftest-1d-$1-report.csv
fi fi
gnuplot -e "filename='perftest-1d-report'" -p perftest-csv2png.gnuplot gnuplot -e "filename='perftest-1d-$1-report'" -p perftest-csv2png.gnuplot
} }
today=`date +"%Y%m%d"` today=`date +"%Y%m%d"`
cd /root cd /root
echoInfo "run Performance Test with 1 day data" echoInfo "run Performance Test with 1 day data"
runPerfTest1d runPerfTest1d $1
echoInfo "Generate plot of 1 day data" echoInfo "Generate plot of 1 day data"
generatePerfPlot1d generatePerfPlot1d $1
echoInfo "End of TSDB-Compare 1-day-data Test" echoInfo "End of TSDB-Compare 1-day-data Test"

2
tests/perftest-scripts/runreal-13d-csv.sh
View File

@ -143,7 +143,7 @@ echo "------------------------------------------------------"
echo echo
today=`date +"%Y%m%d"` today=`date +"%Y%m%d"`
echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-report.csv echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-$1-report.csv
#bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086" #bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086"
#bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020" #bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020"

2
tests/perftest-scripts/runreal-1d-csv.sh
View File

@ -143,7 +143,7 @@ echo "------------------------------------------------------"
echo echo
today=`date +"%Y%m%d"` today=`date +"%Y%m%d"`
echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-report.csv echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-$1-report.csv
#bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086" #bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086"
#bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020" #bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020"

10
tests/perftest-scripts/taosdemo-csv2png.gnuplot
View File

@ -2,7 +2,7 @@
reset reset
set terminal png set terminal png
set title "TaosDemo Performance Report" font ",20" set title filename font ",20"
set ylabel "Time in Seconds" set ylabel "Time in Seconds"
@ -14,13 +14,13 @@ set xlabel "Date"
set style data linespoints set style data linespoints
set terminal pngcairo size 1024,768 enhanced font 'Segoe UI, 10' set terminal pngcairo size 1024,768 enhanced font 'Segoe UI, 10'
set output 'taosdemo-report.png' set output filename . '.png'
set datafile separator ',' set datafile separator ','
set key reverse Left outside set key reverse Left outside
set grid set grid
plot 'taosdemo-report.csv' using 1:2 title "Create 10,000 Table", \ plot filename . '.csv' using 1:2 title "Create 10,000 Tables", \
"" using 1:3 title "Create 10,000 Table and Insert 100,000 data", \ "" using 1:3 title "Delete 10,000 Tables", \
"" using 1:4 title "Request Per Second of Insert 100,000 data" "" using 1:4 title "Create 10,000 Tables and Insert 100,000 records"