Merge branch 'feature/sangshuduo/perftest-script-improve' of https://github.com/taosdata/TDengine into feature/sangshuduo/perftest-script-improve
This commit is contained in:
Shuduo Sang
commit
6d43eaf9f3
|
|
@ -0,0 +1,196 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
today=`date +"%Y%m%d"`
|
||||||
|
TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.cover
|
||||||
|
TDENGINE_COVERAGE_REPORT=$TDENGINE_DIR/tests/coverage-report-$today.log
|
||||||
|
|
||||||
|
# Color setting
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[1;32m'
|
||||||
|
GREEN_DARK='\033[0;32m'
|
||||||
|
GREEN_UNDERLINE='\033[4;32m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
function buildTDengine {
|
||||||
|
echo "check if TDengine need build"
|
||||||
|
cd $TDENGINE_DIR
|
||||||
|
git remote prune origin > /dev/null
|
||||||
|
git remote update > /dev/null
|
||||||
|
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
echo " LOCAL: $LOCAL_COMMIT"
|
||||||
|
echo "REMOTE: $REMOTE_COMMIT"
|
||||||
|
|
||||||
|
# reset counter
|
||||||
|
lcov -d . --zerocounters
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR/debug
|
||||||
|
|
||||||
|
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
|
||||||
|
echo "repo up-to-date"
|
||||||
|
else
|
||||||
|
echo "repo need to pull"
|
||||||
|
git reset --hard
|
||||||
|
git pull
|
||||||
|
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
|
||||||
|
rm -rf *
|
||||||
|
cmake -DCOVER=true -DRANDOM_FILE_FAIL=true .. > /dev/null
|
||||||
|
make > /dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
make install > /dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
function runGeneralCaseOneByOne {
|
||||||
|
while read -r line; do
|
||||||
|
if [[ $line =~ ^./test.sh* ]]; then
|
||||||
|
general_case=`echo $line | grep -w general`
|
||||||
|
|
||||||
|
if [ -n "$general_case" ]; then
|
||||||
|
case=`echo $line |grep general| awk '{print $NF}'`
|
||||||
|
./test.sh -f $case > /dev/null 2>&1 && \
|
||||||
|
echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \
|
||||||
|
echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done < $1
|
||||||
|
}
|
||||||
|
|
||||||
|
function runTest {
|
||||||
|
echo "run Test"
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR/tests/script
|
||||||
|
|
||||||
|
[ -d ../../sim ] && rm -rf ../../sim
|
||||||
|
[ -f $TDENGINE_COVERAGE_REPORT ] && rm $TDENGINE_COVERAGE_REPORT
|
||||||
|
|
||||||
|
runGeneralCaseOneByOne jenkins/basic.txt
|
||||||
|
|
||||||
|
totalSuccess=`grep 'success' $TDENGINE_COVERAGE_REPORT | wc -l`
|
||||||
|
|
||||||
|
if [ "$totalSuccess" -gt "0" ]; then
|
||||||
|
echo -e "\n${GREEN} ### Total $totalSuccess coverage test case(s) succeed! ### ${NC}" | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
totalFailed=`grep 'failed\|fault' $TDENGINE_COVERAGE_REPORT | wc -l`
|
||||||
|
if [ "$totalFailed" -ne "0" ]; then
|
||||||
|
echo -e "${RED} ### Total $totalFailed coverage test case(s) failed! ### ${NC}\n" | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
# exit $totalPyFailed
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR/tests
|
||||||
|
rm -rf ../sim
|
||||||
|
./test-all.sh full python | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
|
||||||
|
# Test Connector
|
||||||
|
stopTaosd
|
||||||
|
$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
|
||||||
|
sleep 10
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR/src/connector/jdbc
|
||||||
|
mvn clean package
|
||||||
|
mvn test | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
|
||||||
|
# Test C Demo
|
||||||
|
stopTaosd
|
||||||
|
$TDENGINE_DIR/debug/build/bin/taosd -c $TDENGINE_DIR/debug/test/cfg > /dev/null &
|
||||||
|
sleep 10
|
||||||
|
yes | $TDENGINE_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
|
||||||
|
# Test waltest
|
||||||
|
dataDir=`grep dataDir $TDENGINE_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
|
||||||
|
walDir=`find $dataDir -name "wal"|head -n1`
|
||||||
|
echo "dataDir: $dataDir" | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
echo "walDir: $walDir" | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
if [ -n "$walDir" ]; then
|
||||||
|
yes | $TDENGINE_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
# run Unit Test
|
||||||
|
echo "Run Unit Test: utilTest, queryTest and cliTest"
|
||||||
|
$TDENGINE_DIR/debug/build/bin/utilTest > /dev/null && echo "utilTest pass!" || echo "utilTest failed!"
|
||||||
|
$TDENGINE_DIR/debug/build/bin/queryTest > /dev/null && echo "queryTest pass!" || echo "queryTest failed!"
|
||||||
|
$TDENGINE_DIR/debug/build/bin/cliTest > /dev/null && echo "cliTest pass!" || echo "cliTest failed!"
|
||||||
|
|
||||||
|
stopTaosd
|
||||||
|
}
|
||||||
|
|
||||||
|
function lcovFunc {
|
||||||
|
echo "collect data by lcov"
|
||||||
|
cd $TDENGINE_DIR
|
||||||
|
|
||||||
|
# collect data
|
||||||
|
lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
|
||||||
|
|
||||||
|
# remove exclude paths
|
||||||
|
lcov --remove coverage.info \
|
||||||
|
'*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \
|
||||||
|
--rc lcov_branch_coverage=1 -o coverage.info
|
||||||
|
|
||||||
|
# generate result
|
||||||
|
lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
|
||||||
|
# push result to coveralls.io
|
||||||
|
coveralls-lcov coverage.info | tee -a $TDENGINE_COVERAGE_REPORT
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendReport {
|
||||||
|
echo "send report"
|
||||||
|
receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
|
||||||
|
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR
|
||||||
|
|
||||||
|
sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_COVERAGE_REPORT
|
||||||
|
BODY_CONTENT=`cat $TDENGINE_COVERAGE_REPORT`
|
||||||
|
echo -e "to: ${receiver}\nsubject: Coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
|
||||||
|
(cat - && uuencode $TDENGINE_COVERAGE_REPORT coverage-report-$today.log) | \
|
||||||
|
ssmtp "${receiver}" && echo "Report Sent!"
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopTaosd {
|
||||||
|
echo "Stop taosd"
|
||||||
|
systemctl stop taosd
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
while [ -n "$PID" ]
|
||||||
|
do
|
||||||
|
pkill -TERM -x taosd
|
||||||
|
sleep 1
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function runTestRandomFail {
|
||||||
|
exec_random_fail_sh=$1
|
||||||
|
default_exec_sh=$TDENGINE_DIR/tests/script/sh/exec.sh
|
||||||
|
[ -f $exec_random_fail_sh ] && cp $exec_random_fail_sh $default_exec_sh || exit 1
|
||||||
|
|
||||||
|
dnodes_random_fail_py=$TDENGINE_DIR/tests/pytest/util/dnodes-no-random-fail.py
|
||||||
|
default_dnodes_py=$TDENGINE_DIR/tests/pytest/util/dnodes.py
|
||||||
|
[ -f $dnodes_random_fail_py ] && cp $dnodes_random_fail_py $default_dnodes_py || exit 1
|
||||||
|
|
||||||
|
runTest NoRandomFail
|
||||||
|
}
|
||||||
|
|
||||||
|
WORK_DIR=/home/shuduo/work/taosdata
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "Run Coverage Test" | tee -a $WORK_DIR/cron.log
|
||||||
|
|
||||||
|
rm /tmp/core-*
|
||||||
|
|
||||||
|
stopTaosd
|
||||||
|
buildTDengine
|
||||||
|
|
||||||
|
runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-random-fail.sh
|
||||||
|
runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-default.sh
|
||||||
|
runTestRandomFail $TDENGINE_DIR/tests/script/sh/exec-no-random-fail.sh
|
||||||
|
|
||||||
|
lcovFunc
|
||||||
|
sendReport
|
||||||
|
stopTaosd
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "End of Coverage Test" | tee -a $WORK_DIR/cron.log
|
||||||
|
|
@ -0,0 +1,191 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
today=`date +"%Y%m%d"`
|
||||||
|
TDENGINE_DIR=/home/shuduo/work/taosdata/TDengine.orig
|
||||||
|
TDENGINE_FULLTEST_REPORT=$TDENGINE_DIR/tests/full-report-$today.log
|
||||||
|
|
||||||
|
# Color setting
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[1;32m'
|
||||||
|
GREEN_DARK='\033[0;32m'
|
||||||
|
GREEN_UNDERLINE='\033[4;32m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
function buildTDengine {
|
||||||
|
echo "check if TDengine need build"
|
||||||
|
|
||||||
|
need_rebuild=false
|
||||||
|
|
||||||
|
if [ ! -d $TDENGINE_DIR ]; then
|
||||||
|
echo "No TDengine source code found!"
|
||||||
|
git clone https://github.com/taosdata/TDengine $TDENGINE_DIR
|
||||||
|
need_rebuild=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR
|
||||||
|
git remote prune origin > /dev/null
|
||||||
|
git remote update > /dev/null
|
||||||
|
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
echo " LOCAL: $LOCAL_COMMIT"
|
||||||
|
echo "REMOTE: $REMOTE_COMMIT"
|
||||||
|
|
||||||
|
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
|
||||||
|
echo "repo up-to-date"
|
||||||
|
else
|
||||||
|
echo "repo need to pull"
|
||||||
|
git pull
|
||||||
|
need_rebuild=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
[ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug
|
||||||
|
cd $TDENGINE_DIR/debug
|
||||||
|
[ -f $TDENGINE_DIR/debug/build/bin/taosd ] || need_rebuild=true
|
||||||
|
|
||||||
|
if $need_rebuild ; then
|
||||||
|
echo "rebuild.."
|
||||||
|
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
rm -rf *
|
||||||
|
cmake .. > /dev/null
|
||||||
|
make > /dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
make install > /dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
function runGeneralCaseOneByOne {
|
||||||
|
while read -r line; do
|
||||||
|
if [[ $line =~ ^./test.sh* ]]; then
|
||||||
|
general_case=`echo $line | grep -w general`
|
||||||
|
|
||||||
|
if [ -n "$general_case" ]; then
|
||||||
|
case=`echo $line | awk '{print $NF}'`
|
||||||
|
|
||||||
|
start_time=`date +%s`
|
||||||
|
./test.sh -f $case > /dev/null 2>&1 && ret=0 || ret = 1
|
||||||
|
end_time=`date +%s`
|
||||||
|
|
||||||
|
if [[ ret -eq 0 ]]; then
|
||||||
|
echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_FULLTEST_REPORT
|
||||||
|
else
|
||||||
|
casename=`echo $case|sed 's/\//\-/g'`
|
||||||
|
find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} +
|
||||||
|
echo -e "${RED}$case failed and log saved${NC}" | tee -a $TDENGINE_FULLTEST_REPORT
|
||||||
|
fi
|
||||||
|
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_FULLTEST_REPORT
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done < $1
|
||||||
|
}
|
||||||
|
|
||||||
|
function runPyCaseOneByOne {
|
||||||
|
while read -r line; do
|
||||||
|
if [[ $line =~ ^python.* ]]; then
|
||||||
|
if [[ $line != *sleep* ]]; then
|
||||||
|
case=`echo $line|awk '{print $NF}'`
|
||||||
|
start_time=`date +%s`
|
||||||
|
$line > /dev/null 2>&1 && ret=0 || ret=1
|
||||||
|
end_time=`date +%s`
|
||||||
|
|
||||||
|
if [[ ret -eq 0 ]]; then
|
||||||
|
echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log
|
||||||
|
else
|
||||||
|
casename=`echo $case|sed 's/\//\-/g'`
|
||||||
|
find $TDENGINE_DIR/sim -name "*log" -exec tar czf $TDENGINE_DIR/fulltest-$today-$casename.log.tar.gz {} +
|
||||||
|
echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log
|
||||||
|
fi
|
||||||
|
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
|
||||||
|
else
|
||||||
|
$line > /dev/null 2>&1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done < $1
|
||||||
|
}
|
||||||
|
|
||||||
|
function runTest {
|
||||||
|
echo "Run Test"
|
||||||
|
cd $TDENGINE_DIR/tests/script
|
||||||
|
|
||||||
|
[ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim
|
||||||
|
[ -f $TDENGINE_FULLTEST_REPORT ] && rm $TDENGINE_FULLTEST_REPORT
|
||||||
|
|
||||||
|
runGeneralCaseOneByOne jenkins/basic.txt
|
||||||
|
|
||||||
|
totalSuccess=`grep 'success' $TDENGINE_FULLTEST_REPORT | wc -l`
|
||||||
|
|
||||||
|
if [ "$totalSuccess" -gt "0" ]; then
|
||||||
|
echo -e "\n${GREEN} ### Total $totalSuccess SIM case(s) succeed! ### ${NC}" \
|
||||||
|
| tee -a $TDENGINE_FULLTEST_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
totalFailed=`grep 'failed\|fault' $TDENGINE_FULLTEST_REPORT | wc -l`
|
||||||
|
if [ "$totalFailed" -ne "0" ]; then
|
||||||
|
echo -e "${RED} ### Total $totalFailed SIM case(s) failed! ### ${NC}\n" \
|
||||||
|
| tee -a $TDENGINE_FULLTEST_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR/tests/pytest
|
||||||
|
[ -d $TDENGINE_DIR/sim ] && rm -rf $TDENGINE_DIR/sim
|
||||||
|
[ -f pytest-out.log ] && rm -f pytest-out.log
|
||||||
|
runPyCaseOneByOne fulltest.sh
|
||||||
|
|
||||||
|
totalPySuccess=`grep 'success' pytest-out.log | wc -l`
|
||||||
|
totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
|
||||||
|
|
||||||
|
cat pytest-out.log >> $TDENGINE_FULLTEST_REPORT
|
||||||
|
if [ "$totalPySuccess" -gt "0" ]; then
|
||||||
|
echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \
|
||||||
|
| tee -a $TDENGINE_FULLTEST_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$totalPyFailed" -ne "0" ]; then
|
||||||
|
echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \
|
||||||
|
| tee -a $TDENGINE_FULLTEST_REPORT
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendReport {
|
||||||
|
echo "Send Report"
|
||||||
|
receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
|
||||||
|
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR/tests
|
||||||
|
|
||||||
|
sed -i 's/\x1b\[[0-9;]*m//g' $TDENGINE_FULLTEST_REPORT
|
||||||
|
BODY_CONTENT=`cat $TDENGINE_FULLTEST_REPORT`
|
||||||
|
|
||||||
|
cd $TDENGINE_DIR
|
||||||
|
tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz
|
||||||
|
|
||||||
|
echo -e "to: ${receiver}\nsubject: Full test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
|
||||||
|
(cat - && uuencode $TDENGINE_FULLTEST_REPORT fulltest-report-$today.log) | \
|
||||||
|
(cat - && uuencode $TDENGINE_DIR/fulltest-$today.tar.gz fulltest-$today.tar.gz) | \
|
||||||
|
ssmtp "${receiver}" && echo "Report Sent!"
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopTaosd {
|
||||||
|
echo "Stop taosd"
|
||||||
|
systemctl stop taosd
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
while [ -n "$PID" ]
|
||||||
|
do
|
||||||
|
pkill -TERM -x taosd
|
||||||
|
sleep 1
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
WORK_DIR=/home/shuduo/work/taosdata
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "Run Full Test" | tee -a $WORK_DIR/cron.log
|
||||||
|
|
||||||
|
stopTaosd
|
||||||
|
buildTDengine
|
||||||
|
runTest
|
||||||
|
sendReport
|
||||||
|
stopTaosd
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "End of Full Test" | tee -a $WORK_DIR/cron.log
|
||||||
|
|
@ -0,0 +1,186 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
today=`date +"%Y%m%d"`
|
||||||
|
TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal.cover
|
||||||
|
TDINTERNAL_COVERAGE_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-coverage-report-$today.log
|
||||||
|
|
||||||
|
# Color setting
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[1;32m'
|
||||||
|
GREEN_DARK='\033[0;32m'
|
||||||
|
GREEN_UNDERLINE='\033[4;32m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
function buildTDinternal {
|
||||||
|
echo "check if TDinternal need build"
|
||||||
|
cd $TDINTERNAL_DIR
|
||||||
|
NEED_COMPILE=0
|
||||||
|
# git remote update
|
||||||
|
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
echo " LOCAL: $LOCAL_COMMIT"
|
||||||
|
echo "REMOTE: $REMOTE_COMMIT"
|
||||||
|
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
|
||||||
|
echo "TDinternal repo is up-to-date"
|
||||||
|
else
|
||||||
|
echo "repo need to pull"
|
||||||
|
# git pull
|
||||||
|
|
||||||
|
# NEED_COMPILE=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
lcov -d . --zerocounters
|
||||||
|
# git submodule update --init --recursive
|
||||||
|
cd $TDINTERNAL_DIR/community
|
||||||
|
TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
|
||||||
|
TDENGINE_LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then
|
||||||
|
echo "community repo is up-to-date"
|
||||||
|
else
|
||||||
|
echo "repo need to pull"
|
||||||
|
# git checkout develop
|
||||||
|
# git pull
|
||||||
|
# NEED_COMPILE=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd $TDINTERNAL_DIR/debug
|
||||||
|
|
||||||
|
if [[ $NEED_COMPILE -eq 1 ]]; then
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
rm -rf *
|
||||||
|
cmake .. > /dev/null
|
||||||
|
make > /dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
make install > /dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
function runUniqueCaseOneByOne {
|
||||||
|
while read -r line; do
|
||||||
|
if [[ $line =~ ^./test.sh* ]]; then
|
||||||
|
case=`echo $line | awk '{print $NF}'`
|
||||||
|
start_time=`date +%s`
|
||||||
|
./test.sh -f $case > /dev/null 2>&1 && \
|
||||||
|
echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT || \
|
||||||
|
echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
end_time=`date +%s`
|
||||||
|
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
fi
|
||||||
|
done < $1
|
||||||
|
}
|
||||||
|
|
||||||
|
function runTest {
|
||||||
|
echo "Run Test"
|
||||||
|
cd $TDINTERNAL_DIR/community/tests/script
|
||||||
|
[ -d ../../sim ] && rm -rf ../../sim
|
||||||
|
|
||||||
|
[ -f $TDINTERNAL_COVERAGE_REPORT ] && rm $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
|
||||||
|
runUniqueCaseOneByOne jenkins/basic.txt
|
||||||
|
|
||||||
|
totalSuccess=`grep 'success' $TDINTERNAL_COVERAGE_REPORT | wc -l`
|
||||||
|
|
||||||
|
if [ "$totalSuccess" -gt "0" ]; then
|
||||||
|
echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
totalFailed=`grep 'failed\|fault' $TDINTERNAL_COVERAGE_REPORT | wc -l`
|
||||||
|
if [ "$totalFailed" -ne "0" ]; then
|
||||||
|
echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
# exit $totalPyFailed
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Test Python test case
|
||||||
|
cd $TDINTERNAL_DIR/community/tests
|
||||||
|
/usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
|
||||||
|
# Test Connector
|
||||||
|
stopTaosd
|
||||||
|
$TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null &
|
||||||
|
sleep 10
|
||||||
|
|
||||||
|
cd $TDINTERNAL_DIR/community/src/connector/jdbc
|
||||||
|
mvn clean package
|
||||||
|
mvn test | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
|
||||||
|
# Test C Demo
|
||||||
|
stopTaosd
|
||||||
|
$TDINTERNAL_DIR/debug/build/bin/taosd -c $TDINTERNAL_DIR/debug/test/cfg > /dev/null &
|
||||||
|
sleep 10
|
||||||
|
yes | $TDINTERNAL_DIR/debug/build/bin/demo 127.0.0.1 | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
|
||||||
|
# Test waltest
|
||||||
|
dataDir=`grep dataDir $TDINTERNAL_DIR/debug/test/cfg/taos.cfg|awk '{print $2}'`
|
||||||
|
walDir=`find $dataDir -name "wal"|head -n1`
|
||||||
|
echo "dataDir: $dataDir\nwalDir: $walDir" | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
if [ -n "$walDir" ]; then
|
||||||
|
yes | $TDINTERNAL_DIR/debug/build/bin/waltest -p $walDir | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
stopTaosd
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendReport {
|
||||||
|
echo "Send Report"
|
||||||
|
receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
|
||||||
|
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
|
||||||
|
|
||||||
|
cd $TDINTERNAL_DIR
|
||||||
|
|
||||||
|
sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
|
||||||
|
BODY_CONTENT=`cat $TDINTERNAL_COVERAGE_REPORT`
|
||||||
|
echo -e "to: ${receiver}\nsubject: TDinternal coverage test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
|
||||||
|
(cat - && uuencode tdinternal-coverage-report-$today.tar.gz tdinternal-coverage-report-$today.tar.gz) | \
|
||||||
|
(cat - && uuencode $TDINTERNAL_COVERAGE_REPORT tdinternal-coverage-report-$today.log) | \
|
||||||
|
ssmtp "${receiver}" && echo "Report Sent!"
|
||||||
|
}
|
||||||
|
|
||||||
|
function lcovFunc {
|
||||||
|
echo "collect data by lcov"
|
||||||
|
cd $TDINTERNAL_DIR
|
||||||
|
|
||||||
|
sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
# collect data
|
||||||
|
lcov -d . --capture --rc lcov_branch_coverage=1 --rc genhtmml_branch_coverage=1 --no-external -b $TDINTERNAL_DIR -o coverage.info
|
||||||
|
|
||||||
|
# remove exclude paths
|
||||||
|
lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' '*/taosdef.h' \
|
||||||
|
--rc lcov_branch_coverage=1 -o coverage.info
|
||||||
|
|
||||||
|
# generate result
|
||||||
|
lcov -l --rc lcov_branch_coverage=1 coverage.info | tee -a $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
|
||||||
|
genhtml -o html coverage.info
|
||||||
|
|
||||||
|
tar czf tdinternal-coverage-report-$today.tar.gz html coverage.info $TDINTERNAL_COVERAGE_REPORT
|
||||||
|
# push result to coveralls.io
|
||||||
|
# coveralls-lcov coverage.info | tee -a tdinternal-coverage-report-$today.log
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopTaosd {
|
||||||
|
echo "Stop taosd"
|
||||||
|
systemctl stop taosd
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
while [ -n "$PID" ]
|
||||||
|
do
|
||||||
|
pkill -TERM -x taosd
|
||||||
|
sleep 1
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
WORK_DIR=/home/shuduo/work/taosdata
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "Run Coverage Test for TDinternal" | tee -a $WORK_DIR/cron.log
|
||||||
|
|
||||||
|
stopTaosd
|
||||||
|
buildTDinternal
|
||||||
|
runTest
|
||||||
|
lcovFunc
|
||||||
|
sendReport
|
||||||
|
stopTaosd
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "End of TDinternal Coverage Test" | tee -a $WORK_DIR/cron.log
|
||||||
|
|
@ -0,0 +1,179 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
today=`date +"%Y%m%d"`
|
||||||
|
TDINTERNAL_DIR=/home/shuduo/work/taosdata/TDinternal
|
||||||
|
TDINTERNAL_TEST_REPORT=$TDINTERNAL_DIR/community/tests/tdinternal-report-$today.log
|
||||||
|
|
||||||
|
# Color setting
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[1;32m'
|
||||||
|
GREEN_DARK='\033[0;32m'
|
||||||
|
GREEN_UNDERLINE='\033[4;32m'
|
||||||
|
NC='\033[0m'
|
||||||
|
|
||||||
|
function buildTDinternal {
|
||||||
|
echo "check if TDinternal need build"
|
||||||
|
cd $TDINTERNAL_DIR
|
||||||
|
NEED_COMPILE=0
|
||||||
|
# git remote update
|
||||||
|
REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
echo " LOCAL: $LOCAL_COMMIT"
|
||||||
|
echo "REMOTE: $REMOTE_COMMIT"
|
||||||
|
if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then
|
||||||
|
echo "TDinternal repo is up-to-date"
|
||||||
|
else
|
||||||
|
echo "repo need to pull"
|
||||||
|
# git pull
|
||||||
|
|
||||||
|
# NEED_COMPILE=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# git submodule update --init --recursive
|
||||||
|
cd $TDINTERNAL_DIR/community
|
||||||
|
TDENGINE_REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop`
|
||||||
|
TDENGINE_LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
if [ "$TDENGINE_LOCAL_COMMIT" == "$TDENGINE_REMOTE_COMMIT" ]; then
|
||||||
|
echo "community repo is up-to-date"
|
||||||
|
else
|
||||||
|
echo "repo need to pull"
|
||||||
|
# git checkout develop
|
||||||
|
# git pull
|
||||||
|
# NEED_COMPILE=1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd $TDINTERNAL_DIR/debug
|
||||||
|
|
||||||
|
if [[ $NEED_COMPILE -eq 1 ]]; then
|
||||||
|
LOCAL_COMMIT=`git rev-parse --short @`
|
||||||
|
rm -rf *
|
||||||
|
cmake .. > /dev/null
|
||||||
|
make > /dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
make install > /dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
function runUniqueCaseOneByOne {
|
||||||
|
while read -r line; do
|
||||||
|
if [[ $line =~ ^./test.sh* ]]; then
|
||||||
|
case=`echo $line | awk '{print $NF}'`
|
||||||
|
start_time=`date +%s`
|
||||||
|
./test.sh -f $case > /dev/null 2>&1 && \
|
||||||
|
echo -e "${GREEN}$case success${NC}" | tee -a $TDINTERNAL_TEST_REPORT || \
|
||||||
|
echo -e "${RED}$case failed${NC}" | tee -a $TDINTERNAL_TEST_REPORT
|
||||||
|
end_time=`date +%s`
|
||||||
|
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDINTERNAL_TEST_REPORT
|
||||||
|
fi
|
||||||
|
done < $1
|
||||||
|
}
|
||||||
|
|
||||||
|
function runPyCaseOneByOne {
|
||||||
|
while read -r line; do
|
||||||
|
if [[ $line =~ ^python.* ]]; then
|
||||||
|
if [[ $line != *sleep* ]]; then
|
||||||
|
case=`echo $line|awk '{print $NF}'`
|
||||||
|
start_time=`date +%s`
|
||||||
|
$line > /dev/null 2>&1 && ret=0 || ret=1
|
||||||
|
end_time=`date +%s`
|
||||||
|
|
||||||
|
if [[ ret -eq 0 ]]; then
|
||||||
|
echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log
|
||||||
|
else
|
||||||
|
casename=`echo $case|sed 's/\//\-/g'`
|
||||||
|
find $TDINTERNAL_DIR/community/sim -name "*log" -exec tar czf $TDINTERNAL_DIR/fulltest-$today-$casename.log.tar.gz {} +
|
||||||
|
echo -e "${RED}$case failed and log saved${NC}" | tee -a pytest-out.log
|
||||||
|
fi
|
||||||
|
echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log
|
||||||
|
else
|
||||||
|
$line > /dev/null 2>&1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done < $1
|
||||||
|
}
|
||||||
|
|
||||||
|
function runTest {
|
||||||
|
echo "Run Test"
|
||||||
|
cd $TDINTERNAL_DIR/community/tests/script
|
||||||
|
[ -d $TDINTERNAL_DIR/sim ] && rm -rf $TDINTERNAL_DIR/sim
|
||||||
|
|
||||||
|
[ -f $TDINTERNAL_TEST_REPORT ] && rm $TDINTERNAL_TEST_REPORT
|
||||||
|
|
||||||
|
runUniqueCaseOneByOne jenkins/basic.txt
|
||||||
|
|
||||||
|
totalSuccess=`grep 'success' $TDINTERNAL_TEST_REPORT | wc -l`
|
||||||
|
|
||||||
|
if [ "$totalSuccess" -gt "0" ]; then
|
||||||
|
echo -e "\n${GREEN} ### Total $totalSuccess TDinternal case(s) succeed! ### ${NC}" | tee -a $TDINTERNAL_TEST_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
totalFailed=`grep 'failed\|fault' $TDINTERNAL_TEST_REPORT | wc -l`
|
||||||
|
if [ "$totalFailed" -ne "0" ]; then
|
||||||
|
echo -e "${RED} ### Total $totalFailed TDinternal case(s) failed! ### ${NC}\n" | tee -a $TDINTERNAL_TEST_REPORT
|
||||||
|
# exit $totalPyFailed
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd $TDINTERNAL_DIR/community/tests/pytest
|
||||||
|
[ -d $TDINTERNAL_DIR/community/sim ] && rm -rf $TDINTERNAL_DIR/community/sim
|
||||||
|
[ -f pytest-out.log ] && rm -f pytest-out.log
|
||||||
|
|
||||||
|
/usr/bin/time -f "Total spent: %e" ./test-all.sh full python | tee -a $TDINTERNAL_TEST_REPORT
|
||||||
|
runPyCaseOneByOne fulltest.sh
|
||||||
|
|
||||||
|
totalPySuccess=`grep 'success' pytest-out.log | wc -l`
|
||||||
|
totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
|
||||||
|
|
||||||
|
cat pytest-out.log >> $TDINTERNAL_TEST_REPORT
|
||||||
|
if [ "$totalPySuccess" -gt "0" ]; then
|
||||||
|
echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" \
|
||||||
|
| tee -a $TDINTERNAL_TEST_REPORT
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$totalPyFailed" -ne "0" ]; then
|
||||||
|
echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" \
|
||||||
|
| tee -a $TDINTERNAL_TEST_REPORT
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendReport {
|
||||||
|
echo "Send Report"
|
||||||
|
receiver="sdsang@taosdata.com, sangshuduo@gmail.com, pxiao@taosdata.com"
|
||||||
|
mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n"
|
||||||
|
|
||||||
|
cd $TDINTERNAL_DIR
|
||||||
|
|
||||||
|
sed -i 's/\x1b\[[0-9;]*m//g' $TDINTERNAL_TEST_REPORT
|
||||||
|
BODY_CONTENT=`cat $TDINTERNAL_TEST_REPORT`
|
||||||
|
|
||||||
|
cd $TDINTERNAL_DIR
|
||||||
|
tar czf fulltest-$today.tar.gz fulltest-$today-*.log.tar.gz
|
||||||
|
|
||||||
|
echo -e "to: ${receiver}\nsubject: TDinternal test report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
|
||||||
|
(cat - && uuencode $TDINTERNAL_TEST_REPORT tdinternal-report-$today.log) | \
|
||||||
|
ssmtp "${receiver}" && echo "Report Sent!"
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopTaosd {
|
||||||
|
echo "Stop taosd"
|
||||||
|
systemctl stop taosd
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
while [ -n "$PID" ]
|
||||||
|
do
|
||||||
|
pkill -KILL -x taosd
|
||||||
|
sleep 1
|
||||||
|
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
WORK_DIR=/home/shuduo/work/taosdata
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "Run Test for TDinternal" | tee -a $WORK_DIR/cron.log
|
||||||
|
|
||||||
|
buildTDinternal
|
||||||
|
runTest
|
||||||
|
sendReport
|
||||||
|
stopTaosd
|
||||||
|
|
||||||
|
date >> $WORK_DIR/cron.log
|
||||||
|
echo "End of TDinternal Test" | tee -a $WORK_DIR/cron.log
|
||||||
Loading…
Reference in New Issue