diff --git a/tests/run_all_ci_cases.sh b/tests/run_all_ci_cases.sh index 959af66d19..11670800b8 100644 --- a/tests/run_all_ci_cases.sh +++ b/tests/run_all_ci_cases.sh @@ -7,12 +7,120 @@ GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -TDENGINE_DIR=/root/TDinternal/community +function print_color() { + local color="$1" + local message="$2" + echo -e "${color}${message}${NC}" +} + +# 初始化参数 +TDENGINE_DIR="/root/TDinternal/community" +BRANCH="" +SAVE_LOG="notsave" + +# 解析命令行参数 +while getopts "hd:b:t:s:" arg; do + case $arg in + d) + TDENGINE_DIR=$OPTARG + ;; + b) + BRANCH=$OPTARG + ;; + s) + SAVE_LOG=$OPTARG + ;; + h) + echo "Usage: $(basename $0) -d [TDengine_dir] -b [branch] -s [save ci case log]" + echo " -d [TDengine_dir] [default /root/TDinternal/community] " + echo " -b [branch] [default local branch] " + echo " -s [save/notsave] [default save ci case log in TDengine_dir/tests/ci_bak] " + exit 0 + ;; + ?) + echo "Usage: ./$(basename $0) -h" + exit 1 + ;; + esac +done + +# 检查是否提供了命令名称 +if [ -z "$TDENGINE_DIR" ]; then + echo "Error: TDengine dir is required." + echo "Usage: $(basename $0) -d [TDengine_dir] -b [branch] -s [save ci case log] " + echo " -d [TDengine_dir] [default /root/TDinternal/community] " + echo " -b [branch] [default local branch] " + echo " -s [save/notsave] [default save ci case log in TDengine_dir/tests/ci_bak] " + exit 1 +fi -#echo "TDENGINE_DIR = $TDENGINE_DIR" +echo "TDENGINE_DIR = $TDENGINE_DIR" today=`date +"%Y%m%d"` -TDENGINE_ALLCI_REPORT=$TDENGINE_DIR/tests/all-ci-report-$today.log +TDENGINE_ALLCI_REPORT="$TDENGINE_DIR/tests/all-ci-report-$today.log" +BACKUP_DIR="$TDENGINE_DIR/tests/ci_bak" +mkdir -p "$BACKUP_DIR" +#cd $BACKUP_DIR && rm -rf * + + +function buildTDengine() { + print_color "$GREEN" "TDengine build start" + + # pull parent code + cd "$TDENGINE_DIR/../" + print_color "$GREEN" "git pull parent code..." + git remote prune origin > /dev/null + git remote update > /dev/null + + # pull tdengine code + cd $TDENGINE_DIR + print_color "$GREEN" "git pull tdengine code..." + git remote prune origin > /dev/null + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch` + LOCAL_COMMIT=`git rev-parse --short @` + print_color "$GREEN" " LOCAL: $LOCAL_COMMIT" + print_color "$GREEN" "REMOTE: $REMOTE_COMMIT" + + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + print_color "$GREEN" "repo up-to-date" + else + print_color "$GREEN" "repo need to pull" + fi + + git reset --hard + git checkout -- . + git checkout $branch + git checkout -- . + git clean -f + git pull + + [ -d $TDENGINE_DIR/debug ] || mkdir $TDENGINE_DIR/debug + cd $TDENGINE_DIR/debug + + print_color "$GREEN" "rebuild.." + LOCAL_COMMIT=`git rev-parse --short @` + + rm -rf * + makecmd="cmake -DBUILD_TEST=false -DBUILD_HTTP=false -DBUILD_DEPENDENCY_TESTS=0 -DBUILD_TOOLS=true -DBUILD_GEOS=true -DBUILD_TEST=true -DBUILD_CONTRIB=false ../../" + print_color "$GREEN" "$makecmd" + $makecmd + + make -j 8 install + + print_color "$GREEN" "TDengine build end" +} + + +# 检查并获取分支名称 +if [ -n "$BRANCH" ]; then + branch="$BRANCH" + print_color "$GREEN" "Testing branch: $branch " + print_color "$GREEN" "Build is required for this test!" + buildTDengine +else + print_color "$GREEN" "Build is not required for this test!" +fi function runCasesOneByOne () { @@ -20,23 +128,50 @@ function runCasesOneByOne () { if [[ "$line" != "#"* ]]; then cmd=`echo $line | cut -d',' -f 5` if [[ "$2" == "sim" ]] && [[ $line == *"script"* ]]; then + echo $cmd case=`echo $cmd | cut -d' ' -f 3` + case_file=`echo $case | tr -d ' /' ` start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT \ - || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ + echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ + echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + + # # 记录日志和备份 + # mkdir -p "$BACKUP_DIR/$case_file" + # tar --exclude='*.sock*' -czf "$BACKUP_DIR/$case_file/sim.tar.gz" -C "$TDENGINE_DIR/.." sim + # mv "$TDENGINE_DIR/tests/$case_file.log" "$BACKUP_DIR/$case_file" + + if [ "$SAVE_LOG" == "save" ]; then + mkdir -p "$BACKUP_DIR/$case_file" + tar --exclude='*.sock*' -czf "$BACKUP_DIR/$case_file/sim.tar.gz" -C "$TDENGINE_DIR/.." sim + mv "$TDENGINE_DIR/tests/$case_file.log" "$BACKUP_DIR/$case_file" + else + echo "This case not save log!" + fi + end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT - + elif [[ "$line" == *"$2"* ]]; then + echo $cmd if [[ "$cmd" == *"pytest.sh"* ]]; then cmd=`echo $cmd | cut -d' ' -f 2-20` fi - case=`echo $cmd | cut -d' ' -f 4-20` + case=`echo $cmd | cut -d' ' -f 4-20` + case_file=`echo $case | tr -d ' /' ` start_time=`date +%s` - date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > /dev/null 2>&1 && \ + date +%F\ %T | tee -a $TDENGINE_ALLCI_REPORT && timeout 20m $cmd > $TDENGINE_DIR/tests/$case_file.log 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_ALLCI_REPORT || \ echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_ALLCI_REPORT + + if [ "$SAVE_LOG" == "save" ]; then + mkdir -p "$BACKUP_DIR/$case_file" + tar --exclude='*.sock*' -czf "$BACKUP_DIR/$case_file/sim.tar.gz" -C "$TDENGINE_DIR/.." sim + mv "$TDENGINE_DIR/tests/$case_file.log" "$BACKUP_DIR/$case_file" + else + echo "This case not save log!" + fi + end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a $TDENGINE_ALLCI_REPORT fi @@ -45,62 +180,62 @@ function runCasesOneByOne () { } function runUnitTest() { - echo "=== Run unit test case ===" - echo " $TDENGINE_DIR/debug" - cd $TDENGINE_DIR/debug + print_color "$GREEN" "=== Run unit test case ===" + print_color "$GREEN" " $TDENGINE_DIR/../debug" + cd $TDENGINE_DIR/../debug ctest -j12 - echo "3.0 unit test done" + print_color "$GREEN" "3.0 unit test done" } function runSimCases() { - echo "=== Run sim cases ===" + print_color "$GREEN" "=== Run sim cases ===" cd $TDENGINE_DIR/tests/script - runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/cases-test.task sim + runCasesOneByOne $TDENGINE_DIR/tests/parallel_test/cases.task sim totalSuccess=`grep 'sim success' $TDENGINE_ALLCI_REPORT | wc -l` if [ "$totalSuccess" -gt "0" ]; then - echo "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + print_color "$GREEN" "### Total $totalSuccess SIM test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT fi totalFailed=`grep 'sim failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` if [ "$totalFailed" -ne "0" ]; then - echo "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + print_color "$RED" "### Total $totalFailed SIM test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT fi } function runPythonCases() { - echo "=== Run python cases ===" + print_color "$GREEN" "=== Run python cases ===" cd $TDENGINE_DIR/tests/parallel_test - sed -i '/compatibility.py/d' cases-test.task + sed -i '/compatibility.py/d' cases.task # army cd $TDENGINE_DIR/tests/army - runCasesOneByOne ../parallel_test/cases-test.task army + runCasesOneByOne ../parallel_test/cases.task army # system-test cd $TDENGINE_DIR/tests/system-test - runCasesOneByOne ../parallel_test/cases-test.task system-test + runCasesOneByOne ../parallel_test/cases.task system-test # develop-test cd $TDENGINE_DIR/tests/develop-test - runCasesOneByOne ../parallel_test/cases-test.task develop-test + runCasesOneByOne ../parallel_test/cases.task develop-test totalSuccess=`grep 'py success' $TDENGINE_ALLCI_REPORT | wc -l` if [ "$totalSuccess" -gt "0" ]; then - echo "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT + print_color "$GREEN" "### Total $totalSuccess python test case(s) succeed! ###" | tee -a $TDENGINE_ALLCI_REPORT fi totalFailed=`grep 'py failed\|fault' $TDENGINE_ALLCI_REPORT | wc -l` if [ "$totalFailed" -ne "0" ]; then - echo "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT + print_color "$RED" "### Total $totalFailed python test case(s) failed! ###" | tee -a $TDENGINE_ALLCI_REPORT fi } function runTest() { - echo "run Test" + print_color "$GREEN" "run Test" cd $TDENGINE_DIR [ -d sim ] && rm -rf sim @@ -119,20 +254,20 @@ function runTest() { } function stopTaosd { - echo "Stop taosd start" - systemctl stop taosd - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + print_color "$GREEN" "Stop taosd start" + systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - pkill -TERM -x taosd - sleep 1 - PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done - echo "Stop tasod end" + print_color "$GREEN" "Stop tasod end" } function stopTaosadapter { - echo "Stop taosadapter" + print_color "$GREEN" "Stop taosadapter" systemctl stop taosadapter.service PID=`ps -ef|grep -w taosadapter | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] @@ -141,18 +276,18 @@ function stopTaosadapter { sleep 1 PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done - echo "Stop tasoadapter end" + print_color "$GREEN" "Stop tasoadapter end" } WORK_DIR=/root/ date >> $WORK_DIR/date.log -echo "Run ALL CI Test Cases" | tee -a $WORK_DIR/date.log +print_color "$GREEN" "Run all ci test cases" | tee -a $WORK_DIR/date.log stopTaosd runTest date >> $WORK_DIR/date.log -echo "End of CI Test Cases" | tee -a $WORK_DIR/date.log \ No newline at end of file +print_color "$GREEN" "End of ci test cases" | tee -a $WORK_DIR/date.log \ No newline at end of file