xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

update script to support wal param

This commit is contained in:
Shuduo Sang 2020-06-08 11:16:13 +08:00
parent 63bf9970aa
commit fb1f837d28
8 changed files with 124 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/perftest-scripts/perftest-csv2png.gnuplot
View File

@ -2,7 +2,7 @@
reset
set terminal png
set title "Performance Test Report" font ",20"
set title filename font ",20"
set ylabel "Time in Seconds"

72
tests/perftest-scripts/perftest-daily.sh
View File

@ -16,9 +16,26 @@ function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") &&
function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } #
function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } #
function set-Wal {
function setMaxConnections {
echo "/etc/taos/taos.cfg maxConnection will be set to $1"
hasText=`grep "maxConnections" /etc/taos/taos.cfg`
if [[ -z "$hasText" ]]; then
echo "maxConnections $1" >> /etc/taos/taos.cfg
else
sed -i 's/^maxConnections.*$/maxConnections '"$1"'/g' /etc/taos/taos.cfg
fi
}
function setWal {
echo "/etc/taos/taos.cfg walLevel will be set to $1"
hasText=`grep "walLevel" /etc/taos/taos.cfg`
if [[ -z "$hasText" ]]; then
echo "walLevel $1" >> /etc/taos/taos.cfg
else
sed -i 's/^walLevel.*$/walLevel '"$1"'/g' /etc/taos/taos.cfg
fi
}
function collectSysInfo {
@ -70,15 +87,25 @@ function sendReport {
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
echo -e "to: ${receiver}\nsubject: Perf test report ${today}, commit ID: ${LOCAL_COMMIT}\n" | \
(cat - && uuencode perftest-1d-$today.log perftest-1d-$today.log)| \
(cat - && uuencode perftest-1d-report.csv perftest-1d-report-$today.csv) | \
(cat - && uuencode perftest-1d-report.png perftest-1d-report-$today.png) | \
(cat - && uuencode perftest-13d-$today.log perftest-13d-$today.log)| \
(cat - && uuencode perftest-13d-report.csv perftest-13d-report-$today.csv) | \
(cat - && uuencode perftest-13d-report.png perftest-13d-report-$today.png) | \
(cat - && uuencode taosdemo-$today.log taosdemo-$today.log) | \
(cat - && uuencode taosdemo-report.csv taosdemo-report-$today.csv) | \
(cat - && uuencode taosdemo-report.png taosdemo-report-$today.png) | \
(cat - && uuencode perftest-1d-wal1-$today.log perftest-1d-wal1-$today.log)| \
(cat - && uuencode perftest-1d-wal1-report.csv perftest-1d-wal1-report-$today.csv) | \
(cat - && uuencode perftest-1d-wal1-report.png perftest-1d-wal1-report-$today.png) | \
(cat - && uuencode perftest-13d-wal1-$today.log perftest-13d-wal1-$today.log)| \
(cat - && uuencode perftest-13d-wal1-report.csv perftest-13d-wal1-report-$today.csv) | \
(cat - && uuencode perftest-13d-wal1-report.png perftest-13d-wal1-report-$today.png) | \
(cat - && uuencode taosdemo-wal1-$today.log taosdemo-wal1-$today.log) | \
(cat - && uuencode taosdemo-wal1-report.csv taosdemo-wal1-report-$today.csv) | \
(cat - && uuencode taosdemo-rps-wal1-report.csv taosdemo-rps-wal1-report-$today.csv) | \
(cat - && uuencode taosdemo-wal1-report.png taosdemo-wal1-report-$today.png) | \
(cat - && uuencode perftest-1d-wal2-$today.log perftest-1d-wal2-$today.log)| \
(cat - && uuencode perftest-1d-wal2-report.csv perftest-1d-wal2-report-$today.csv) | \
(cat - && uuencode perftest-1d-wal2-report.png perftest-1d-wal2-report-$today.png) | \
(cat - && uuencode perftest-13d-wal2-$today.log perftest-13d-wal2-$today.log)| \
(cat - && uuencode perftest-13d-wal2-report.csv perftest-13d-wal2-report-$today.csv) | \
(cat - && uuencode perftest-13d-wal2-report.png perftest-13d-wal2-report-$today.png) | \
(cat - && uuencode taosdemo-wal2-$today.log taosdemo-wal2-$today.log) | \
(cat - && uuencode taosdemo-wal2-report.csv taosdemo-wal2-report-$today.csv) | \
(cat - && uuencode taosdemo-rps-wal2-report.csv taosdemo-rps-wal2-report-$today.csv) | \
(cat - && uuencode sysinfo.log sysinfo.txt) | \
(cat - && uuencode taos.cfg taos-cfg-$today.txt) | \
ssmtp "${receiver}"
@ -91,17 +118,34 @@ echo -e "cron-ran-at-${today}" >> cron.log
echoInfo "Build TDengine"
buildTDengine
set-Wal "2"
############################
setMaxConnections 100
############################
setWal "2"
cd /root
./perftest-tsdb-compare-1d.sh
./perftest-tsdb-compare-1d.sh "wal2"
cd /root
./perftest-tsdb-compare-13d.sh
./perftest-tsdb-compare-13d.sh "wal2"
cd /root
./perftest-taosdemo.sh
./perftest-taosdemo.sh "wal2"
#############################
setWal "1"
cd /root
./perftest-tsdb-compare-1d.sh "wal1"
cd /root
./perftest-tsdb-compare-13d.sh "wal1"
cd /root
./perftest-taosdemo.sh "wal1"
#############################
collectSysInfo
echoInfo "Send Report"

72
tests/perftest-scripts/perftest-taosdemo.sh
View File

@ -1,20 +1,20 @@
#!/bin/bash
# Coloured Echoes #
function red_echo { echo -e "\033[31m$@\033[0m"; } #
function green_echo { echo -e "\033[32m$@\033[0m"; } #
function yellow_echo { echo -e "\033[33m$@\033[0m"; } #
function white_echo { echo -e "\033[1;37m$@\033[0m"; } #
# Coloured Printfs #
function red_printf { printf "\033[31m$@\033[0m"; } #
function green_printf { printf "\033[32m$@\033[0m"; } #
function yellow_printf { printf "\033[33m$@\033[0m"; } #
function white_printf { printf "\033[1;37m$@\033[0m"; } #
# Debugging Outputs #
function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } #
function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } #
function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } #
function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } #
# Coloured Echoes
function red_echo { echo -e "\033[31m$@\033[0m"; }
function green_echo { echo -e "\033[32m$@\033[0m"; }
function yellow_echo { echo -e "\033[33m$@\033[0m"; }
function white_echo { echo -e "\033[1;37m$@\033[0m"; }
# Coloured Printfs
function red_printf { printf "\033[31m$@\033[0m"; }
function green_printf { printf "\033[32m$@\033[0m"; }
function yellow_printf { printf "\033[33m$@\033[0m"; }
function white_printf { printf "\033[1;37m$@\033[0m"; }
# Debugging Outputs
function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; }
function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; }
function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; }
function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; }
function restartTaosd {
systemctl stop taosd
@ -32,39 +32,57 @@ function runCreateTableOnly {
echoInfo "Restart Taosd"
restartTaosd
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$today.log"
demoTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'`
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -n 0 2>&1 | tee taosdemo-$1-$today.log"
demoCreateTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'`
}
function runDeleteTableOnly {
echoInfo "Restart Taosd"
restartTaosd
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo -t 0 -D 1 2>&1 | tee taosdemo-$1-$today.log"
demoDeleteTableOnly=`grep "Total:" totaltime.out|awk '{print $2}'`
}
function runCreateTableThenInsert {
echoInfo "Restart Taosd"
restartTaosd
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$today.log"
/usr/bin/time -f "Total: %e" -o totaltime.out bash -c "yes | taosdemo 2>&1 | tee -a taosdemo-$1-$today.log"
demoTableAndInsert=`grep "Total:" totaltime.out|awk '{print $2}'`
demoRPS=`grep "records\/second" taosdemo-$today.log | tail -n1 | awk '{print $13}'`
demoRPS=`grep "records\/second" taosdemo-$1-$today.log | tail -n1 | awk '{print $13}'`
}
function generateTaosdemoPlot {
echo "${today}, demoTableOnly: ${demoTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log
echo "${today}, ${demoTableOnly}, ${demoTableAndInsert}, ${demoRPS}" >> taosdemo-report.csv
echo "${today} $1, demoCreateTableOnly: ${demoCreateTableOnly}, demoDeleteTableOnly: ${demoDeleteTableOnly}, demoTableAndInsert: ${demoTableAndInsert}" | tee -a taosdemo-$today.log
echo "${today}, ${demoCreateTableOnly}, ${demoDeleteTableOnly}, ${demoTableAndInsert}">> taosdemo-$1-report.csv
echo "${today}, ${demoRPS}" >> taosdemo-rps-$1-report.csv
csvLines=`cat taosdemo-report.csv | wc -l`
csvLines=`cat taosdemo-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then
sed -i '1d' taosdemo-report.csv
sed -i '1d' taosdemo-$1-report.csv
fi
gnuplot -p taosdemo-csv2png.gnuplot
csvLines=`cat taosdemo-rps-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then
sed -i '1d' taosdemo-rps-$1-report.csv
fi
gnuplot -e "filename='taosdemo-$1-report'" -p taosdemo-csv2png.gnuplot
gnuplot -e "filename='taosdemo-rps-$1-report'" -p taosdemo-rps-csv2png.gnuplot
}
today=`date +"%Y%m%d"`
cd /root
echoInfo "Test Create Table Only "
runCreateTableOnly
runCreateTableOnly $1
echoInfo "Test Create Table then Insert data"
runCreateTableThenInsert
runDeleteTableOnly $1
echoInfo "Test Create Table then Insert data"
runCreateTableThenInsert $1
echoInfo "Generate plot for taosdemo"
generateTaosdemoPlot
generateTaosdemoPlot $1
echoInfo "End of TaosDemo Test"

12
tests/perftest-scripts/perftest-tsdb-compare-13d.sh
View File

@ -33,26 +33,26 @@ function runPerfTest13d {
restartTaosd
cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare
./runreal-13d-csv.sh 2>&1 | tee /root/perftest-13d-$today.log
./runreal-13d-csv.sh $1 2>&1 | tee /root/perftest-13d-$1-$today.log
}
function generatePerfPlot13d {
cd /root
csvLines=`cat perftest-13d-report.csv | wc -l`
csvLines=`cat perftest-13d-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then
sed -i '1d' perftest-13d-report.csv
sed -i '1d' perftest-13d-$1-report.csv
fi
gnuplot -e "filename='perftest-13d-report'" -p perftest-csv2png.gnuplot
gnuplot -e "filename='perftest-13d-$1-report'" -p perftest-csv2png.gnuplot
}
today=`date +"%Y%m%d"`
cd /root
echoInfo "run Performance Test with 13 days data"
runPerfTest13d
runPerfTest13d $1
echoInfo "Generate plot of 13 days data"
generatePerfPlot13d
generatePerfPlot13d $1
echoInfo "End of TSDB-Compare 13-days-data Test"

12
tests/perftest-scripts/perftest-tsdb-compare-1d.sh
View File

@ -33,26 +33,26 @@ function runPerfTest1d {
restartTaosd
cd /home/taos/tliu/timeseriesdatabase-comparisons/build/tsdbcompare
./runreal-1d-csv.sh 2>&1 | tee /root/perftest-1d-$today.log
./runreal-1d-csv.sh $1 2>&1 | tee /root/perftest-1d-$1-$today.log
}
function generatePerfPlot1d {
cd /root
csvLines=`cat perftest-1d-report.csv | wc -l`
csvLines=`cat perftest-1d-$1-report.csv | wc -l`
if [ "$csvLines" -gt "10" ]; then
sed -i '2d' perftest-1d-report.csv
sed -i '2d' perftest-1d-$1-report.csv
fi
gnuplot -e "filename='perftest-1d-report'" -p perftest-csv2png.gnuplot
gnuplot -e "filename='perftest-1d-$1-report'" -p perftest-csv2png.gnuplot
}
today=`date +"%Y%m%d"`
cd /root
echoInfo "run Performance Test with 1 day data"
runPerfTest1d
runPerfTest1d $1
echoInfo "Generate plot of 1 day data"
generatePerfPlot1d
generatePerfPlot1d $1
echoInfo "End of TSDB-Compare 1-day-data Test"

2
tests/perftest-scripts/runreal-13d-csv.sh
View File

@ -143,7 +143,7 @@ echo "------------------------------------------------------"
echo
today=`date +"%Y%m%d"`
echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-report.csv
echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-13d-$1-report.csv
#bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086"
#bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020"

2
tests/perftest-scripts/runreal-1d-csv.sh
View File

@ -143,7 +143,7 @@ echo "------------------------------------------------------"
echo
today=`date +"%Y%m%d"`
echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-report.csv
echo "${today}, ${TDWTM}, ${TDQ1}, ${TDQ2}, ${TDQ3}, ${TDQ4}" >> /root/perftest-1d-$1-report.csv
#bulk_query_gen/bulk_query_gen -format influx-http -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_influxdb/query_benchmarker_influxdb -urls="http://172.26.89.231:8086"
#bulk_query_gen/bulk_query_gen -format tdengine -query-type 1-host-1-hr -scale-var 10 -queries 1000 | query_benchmarker_tdengine/query_benchmarker_tdengine -urls="http://172.26.89.231:6020"

10
tests/perftest-scripts/taosdemo-csv2png.gnuplot
View File

@ -2,7 +2,7 @@
reset
set terminal png
set title "TaosDemo Performance Report" font ",20"
set title filename font ",20"
set ylabel "Time in Seconds"
@ -14,13 +14,13 @@ set xlabel "Date"
set style data linespoints
set terminal pngcairo size 1024,768 enhanced font 'Segoe UI, 10'
set output 'taosdemo-report.png'
set output filename . '.png'
set datafile separator ','
set key reverse Left outside
set grid
plot 'taosdemo-report.csv' using 1:2 title "Create 10,000 Table", \
"" using 1:3 title "Create 10,000 Table and Insert 100,000 data", \
"" using 1:4 title "Request Per Second of Insert 100,000 data"
plot filename . '.csv' using 1:2 title "Create 10,000 Tables", \
"" using 1:3 title "Delete 10,000 Tables", \
"" using 1:4 title "Create 10,000 Tables and Insert 100,000 records"