From 33c85e06c8c2a5c3e618ef44a2a25dfc4869bc11 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Fri, 24 Jun 2022 15:17:17 +0800 Subject: [PATCH 1/2] ci: optimize Jenkins log --- Jenkinsfile2 | 103 +++++++++++++++++++++++------------ tests/parallel_test/run.sh | 109 ++++++++++++++++++++++++++++--------- 2 files changed, 149 insertions(+), 63 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 94af11868a..38af8fd286 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -43,10 +43,8 @@ def pre_test(){ sh ''' cd ${WK} git reset --hard - git fetch || git fetch cd ${WKC} git reset --hard - git fetch || git fetch ''' script { if (env.CHANGE_TARGET == 'master') { @@ -82,6 +80,7 @@ def pre_test(){ if (env.CHANGE_URL =~ /\/TDengine\//) { sh ''' cd ${WKC} + git remote prune origin git pull >/dev/null git log -5 echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log @@ -107,6 +106,7 @@ def pre_test(){ git log -5 echo "tdinternal log merged: `git log -5`" >>${WKDIR}/jenkins.log cd ${WKC} + git remote prune origin git pull >/dev/null git log -5 echo "community log: `git log -5`" >>${WKDIR}/jenkins.log @@ -137,53 +137,51 @@ def pre_test_win(){ set date /t time /t - rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug || exit 0 + rd /s /Q %WIN_INTERNAL_ROOT%\\debug || exit 0 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git reset --hard - git fetch || git fetch ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git reset --hard - git fetch || git fetch ''' script { if (env.CHANGE_TARGET == 'master') { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout master ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout master ''' } else if(env.CHANGE_TARGET == '2.0') { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout 2.0 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout 2.0 ''' } else if(env.CHANGE_TARGET == '3.0') { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout 3.0 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout 3.0 ''' } else { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout develop ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout develop ''' } @@ -191,36 +189,38 @@ def pre_test_win(){ script { if (env.CHANGE_URL =~ /\/TDengine\//) { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% + git remote prune origin git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git fetch origin +refs/pull/%CHANGE_ID%/merge ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git checkout -qf FETCH_HEAD ''' } else if (env.CHANGE_URL =~ /\/TDinternal\//) { bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git fetch origin +refs/pull/%CHANGE_ID%/merge ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git checkout -qf FETCH_HEAD ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% + git remote prune origin git pull ''' } else { @@ -230,27 +230,27 @@ def pre_test_win(){ } } bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% git branch git log -5 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git branch git log -5 ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community + cd %WIN_COMMUNITY_ROOT% git submodule update --init --recursive ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% git branch git reset --hard git pull ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% git log -5 ''' } @@ -258,7 +258,7 @@ def pre_test_build_win() { bat ''' echo "building ..." time /t - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal + cd %WIN_INTERNAL_ROOT% mkdir debug cd debug time /t @@ -273,9 +273,9 @@ def pre_test_build_win() { time /t ''' bat ''' - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% python -m pip install . - xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' return 1 } @@ -283,7 +283,7 @@ def run_win_ctest() { bat ''' echo "windows ctest ..." time /t - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug + cd %WIN_INTERNAL_ROOT%\\debug ctest -j 1 || exit 7 time /t ''' @@ -292,12 +292,12 @@ def run_win_test() { echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}" bat ''' echo "windows test ..." - cd C:\\workspace\\%EXECUTOR_NUMBER%\\taos-connector-python + cd %WIN_CONNECTOR_ROOT% python -m pip install . - xcopy /e/y/i/f C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ls -l C:\\Windows\\System32\\taos.dll time /t - cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDinternal\\community\\tests\\system-test + cd %WIN_SYSTEM_TEST_ROOT% echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''" echo "testing ..." test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}" @@ -319,6 +319,12 @@ pipeline { parallel { stage('windows test') { agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} + environment{ + WIN_INTERNAL_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal" + WIN_COMMUNITY_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community" + WIN_SYSTEM_TEST_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\TDinternal\\community\\tests\\system-test" + WIN_CONNECTOR_ROOT="C:\\workspace\\${env.EXECUTOR_NUMBER}\\taos-connector-python" + } steps { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 55, unit: 'MINUTES'){ @@ -368,11 +374,36 @@ pipeline { rm -f /tmp/cases.task ./collect_cases.sh -e ''' + def extra_param = "" + def log_server_file = "/home/log_server.json" + def timeout_cmd = "" + if (fileExists(log_server_file)) { + def log_server_enabled = sh ( + script: 'jq .enabled ' + log_server_file, + returnStdout: true + ).trim() + def timeout_param = sh ( + script: 'jq .timeout ' + log_server_file, + returnStdout: true + ).trim() + if (timeout_param != "null" && timeout_param != "0") { + timeout_cmd = "timeout " + timeout_param + } + if (log_server_enabled == "1") { + def log_server = sh ( + script: 'jq .server ' + log_server_file + ' | sed "s/\\\"//g"', + returnStdout: true + ).trim() + if (log_server != "null" && log_server != "") { + extra_param = "-w " + log_server + } + } + } sh ''' cd ${WKC}/tests/parallel_test export DEFAULT_RETRY_TIME=2 date - timeout 2100 time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 + ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t /tmp/cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 480 ''' + extra_param + ''' ''' } } diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index e9871637bd..700d6853d2 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -8,11 +8,12 @@ function usage() { echo -e "\t -l log dir" echo -e "\t -e enterprise edition" echo -e "\t -o default timeout value" + echo -e "\t -w log web server" echo -e "\t -h help" } ent=0 -while getopts "m:t:b:l:o:eh" opt; do +while getopts "m:t:b:l:o:w:eh" opt; do case $opt in m) config_file=$OPTARG @@ -32,6 +33,9 @@ while getopts "m:t:b:l:o:eh" opt; do o) timeout_param="-o $OPTARG" ;; + w) + web_server=$OPTARG + ;; h) usage exit 0 @@ -64,10 +68,11 @@ if [ ! -f $t_file ]; then exit 1 fi date_tag=`date +%Y%m%d-%H%M%S` +test_log_dir=${branch}_${date_tag} if [ -z $log_dir ]; then - log_dir="log/${branch}_${date_tag}" + log_dir="log/${test_log_dir}" else - log_dir="$log_dir/${branch}_${date_tag}" + log_dir="$log_dir/${test_log_dir}" fi hosts=() @@ -190,44 +195,54 @@ function run_thread() { # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 + local case_log_file=$log_dir/${case_file}.txt start_time=`date +%s` + local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` + case_index=`printf "%5d" $case_index` + local case_info=`echo "$line"|cut -d, -f 3,4` while [ ${redo_count} -lt 6 ]; do - if [ -f $log_dir/$case_file.log ]; then - cp $log_dir/$case_file.log $log_dir/$case_file.${redo_count}.redolog + if [ -f $case_log_file ]; then + cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt fi - echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$log_dir/$case_file.log - echo -e "\e[33m >>>>> \e[0m ${case_cmd}" - date >>$log_dir/$case_file.log - # $cmd 2>&1 | tee -a $log_dir/$case_file.log + echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$case_log_file + local current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo -e "$case_index \e[33m START >>>>> \e[0m ${case_info} \e[33m[$current_time]\e[0m" + echo "$current_time" >>$case_log_file + local real_start_time=`date +%s` + # $cmd 2>&1 | tee -a $case_log_file # ret=${PIPESTATUS[0]} - $cmd >>$log_dir/$case_file.log 2>&1 + $cmd >>$case_log_file 2>&1 ret=$? - echo "${hosts[index]} `date` ret:${ret}" >>$log_dir/$case_file.log + local real_end_time=`date +%s` + local time_elapsed=$(( real_end_time - real_start_time )) + echo "execute time: ${time_elapsed}s" >>$case_log_file + current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo "${hosts[index]} $current_time exit code:${ret}" >>$case_log_file if [ $ret -eq 0 ]; then break fi redo=0 - grep -q "wait too long for taosd start" $log_dir/$case_file.log + grep -q "wait too long for taosd start" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "kex_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log + grep -q "kex_exchange_identification: Connection closed by remote host" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "ssh_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log + grep -q "ssh_exchange_identification: Connection closed by remote host" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "kex_exchange_identification: read: Connection reset by peer" $log_dir/$case_file.log + grep -q "kex_exchange_identification: read: Connection reset by peer" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "Database not ready" $log_dir/$case_file.log + grep -q "Database not ready" $case_log_file if [ $? -eq 0 ]; then redo=1 fi - grep -q "Unable to establish connection" $log_dir/$case_file.log + grep -q "Unable to establish connection" $case_log_file if [ $? -eq 0 ]; then redo=1 fi @@ -240,11 +255,18 @@ function run_thread() { redo_count=$(( redo_count + 1 )) done end_time=`date +%s` - echo >>$log_dir/$case_file.log - echo "${hosts[index]} execute time: $(( end_time - start_time ))s" >>$log_dir/$case_file.log + echo >>$case_log_file + total_time=$(( end_time - start_time )) + echo "${hosts[index]} total time: ${total_time}s" >>$case_log_file # echo "$thread_no ${line} DONE" - if [ $ret -ne 0 ]; then - flock -x $lock_file -c "echo \"${hosts[index]} ret:${ret} ${line}\" >>$log_dir/failed.log" + if [ $ret -eq 0 ]; then + echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m" + else + if [ ! -z ${web_server} ]; then + flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" + else + flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" + fi mkdir -p $log_dir/${case_file}.coredump local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump" local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" @@ -253,13 +275,12 @@ function run_thread() { fi cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" $cmd # 2>/dev/null - local case_info=`echo "$line"|cut -d, -f 3,4` local corefile=`ls $log_dir/${case_file}.coredump/` - echo -e "$case_info \e[31m failed\e[0m" + echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m" echo "=========================log============================" - cat $log_dir/$case_file.log + cat $case_log_file echo "=====================================================" - echo -e "\e[34m log file: $log_dir/$case_file.log \e[0m" + echo -e "\e[34m log file: $case_log_file \e[0m" if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} @@ -325,6 +346,10 @@ mkdir -p $log_dir rm -rf $log_dir/* task_file=$log_dir/$$.task lock_file=$log_dir/$$.lock +index_file=$log_dir/case_index.txt +stat_file=$log_dir/stat.txt +failed_case_file=$log_dir/failed.txt +echo "0" >$index_file i=0 j=0 @@ -350,15 +375,45 @@ rm -f $lock_file rm -f $task_file # docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f +echo "=====================================================================" +echo "log dir: $log_dir" +total_cases=`cat $index_file` +failed_cases=0 +if [ -f $failed_case_file ]; then + if [ ! -z "$web_server" ]; then + failed_cases=`grep -v "$web_server" $failed_case_file|wc -l` + else + failed_cases=`grep -v "log file:" $failed_case_file|wc -l` + fi +fi +success_cases=$(( total_cases - failed_cases )) +echo "Total Cases: $total_cases" >$stat_file +echo "Successful: $success_cases" >>$stat_file +echo "Failed: $failed_cases" >>$stat_file +cat $stat_file + RET=0 i=1 -if [ -f "$log_dir/failed.log" ]; then +if [ -f "${failed_case_file}" ]; then echo "=====================================================" while read line; do + if [ ! -z "${web_server}" ]; then + echo "$line"|grep -q "${web_server}" + if [ $? -eq 0 ]; then + echo " $line" + continue + fi + else + echo "$line"|grep -q "log file:" + if [ $? -eq 0 ]; then + echo " $line" + continue + fi + fi line=`echo "$line"|cut -d, -f 3,4` echo -e "$i. $line \e[31m failed\e[0m" >&2 i=$(( i + 1 )) - done <$log_dir/failed.log + done <${failed_case_file} RET=1 fi From 181874339bc32c34cf5c7c30d0854f6a7f9aa8e0 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Fri, 24 Jun 2022 15:46:45 +0800 Subject: [PATCH 2/2] fix: change Jenkins run case timeout --- Jenkinsfile2 | 2 +- tests/parallel_test/run.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 38af8fd286..b65576deaf 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -363,7 +363,7 @@ pipeline { echo "${linux_node_ip}:${linux_node_pass}" } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - timeout(time: 40, unit: 'MINUTES'){ + timeout(time: 120, unit: 'MINUTES'){ pre_test() script { sh ''' diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index 700d6853d2..26f481e571 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -281,6 +281,9 @@ function run_thread() { cat $case_log_file echo "=====================================================" echo -e "\e[34m log file: $case_log_file \e[0m" + if [ ! -z "${web_server}" ]; then + echo "${web_server}/$test_log_dir/${case_file}.txt" + fi if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]}