From 8cdd28131f642a5d087b04e6d3817db8c4377a0e Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Mon, 8 Jan 2024 12:29:09 +0800 Subject: [PATCH] test: save case.sql when case is failed and set default saved sqlfile True --- tests/parallel_test/run.sh | 330 +++++++++++++++++++----------------- tests/system-test/pytest.sh | 12 +- 2 files changed, 176 insertions(+), 166 deletions(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index cf21349513..ef9413a9d5 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -15,61 +15,61 @@ function usage() { ent=0 while getopts "m:t:b:l:o:w:eh" opt; do case $opt in - m) - config_file=$OPTARG - ;; - t) - t_file=$OPTARG - ;; - b) - branch=$OPTARG - ;; - l) - log_dir=$OPTARG - ;; - e) - ent=1 - ;; - o) - timeout_param="-o $OPTARG" - ;; - w) - web_server=$OPTARG - ;; - h) - usage - exit 0 - ;; - \?) - echo "Invalid option: -$OPTARG" - usage - exit 0 - ;; + m) + config_file=$OPTARG + ;; + t) + t_file=$OPTARG + ;; + b) + branch=$OPTARG + ;; + l) + log_dir=$OPTARG + ;; + e) + ent=1 + ;; + o) + timeout_param="-o $OPTARG" + ;; + w) + web_server=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; esac done #config_file=$1 -if [ -z $config_file ]; then +if [ -z "$config_file" ]; then usage exit 1 fi -if [ ! -f $config_file ]; then +if [ ! -f "$config_file" ]; then echo "$config_file not found" usage exit 1 fi #t_file=$2 -if [ -z $t_file ]; then +if [ -z "$t_file" ]; then usage exit 1 fi -if [ ! -f $t_file ]; then +if [ ! -f "$t_file" ]; then echo "$t_file not found" usage exit 1 fi -date_tag=`date +%Y%m%d-%H%M%S` +date_tag=$(date +%Y%m%d-%H%M%S) test_log_dir=${branch}_${date_tag} -if [ -z $log_dir ]; then +if [ -z "$log_dir" ]; then log_dir="log/${test_log_dir}" else log_dir="$log_dir/${test_log_dir}" @@ -82,42 +82,41 @@ workdirs=() threads=() i=0 -while [ 1 ]; do - host=`jq .[$i].host $config_file` +while true; do + host=$(jq .[$i].host "$config_file") if [ "$host" = "null" ]; then break fi - username=`jq .[$i].username $config_file` + username=$(jq .[$i].username "$config_file") if [ "$username" = "null" ]; then break fi - password=`jq .[$i].password $config_file` + password=$(jq .[$i].password "$config_file") if [ "$password" = "null" ]; then password="" fi - workdir=`jq .[$i].workdir $config_file` + workdir=$(jq .[$i].workdir "$config_file") if [ "$workdir" = "null" ]; then break fi - thread=`jq .[$i].thread $config_file` + thread=$(jq .[$i].thread "$config_file") if [ "$thread" = "null" ]; then break fi - hosts[i]=`echo $host|sed 's/\"$//'|sed 's/^\"//'` - usernames[i]=`echo $username|sed 's/\"$//'|sed 's/^\"//'` - passwords[i]=`echo $password|sed 's/\"$//'|sed 's/^\"//'` - workdirs[i]=`echo $workdir|sed 's/\"$//'|sed 's/^\"//'` + hosts[i]=$(echo "$host" | sed 's/\"$//' | sed 's/^\"//') + usernames[i]=$(echo "$username" | sed 's/\"$//' | sed 's/^\"//') + passwords[i]=$(echo "$password" | sed 's/\"$//' | sed 's/^\"//') + workdirs[i]=$(echo "$workdir" | sed 's/\"$//' | sed 's/^\"//') threads[i]=$thread - i=$(( i + 1 )) + i=$((i + 1)) done - function prepare_cases() { - cat $t_file >>$task_file + cat "$t_file" >>"$task_file" local i=0 - while [ $i -lt $1 ]; do - echo "%%FINISHED%%" >>$task_file - i=$(( i + 1 )) + while [ $i -lt "$1" ]; do + echo "%%FINISHED%%" >>"$task_file" + i=$((i + 1)) done } @@ -125,7 +124,7 @@ function clean_tmp() { # clean tmp dir local index=$1 local ssh_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then ssh_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local cmd="${ssh_script} rm -rf ${workdirs[index]}/tmp" @@ -136,7 +135,7 @@ function run_thread() { local index=$1 local thread_no=$2 local runcase_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then runcase_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi local count=0 @@ -147,8 +146,9 @@ function run_thread() { local cmd="${runcase_script} ${script}" # script="echo" - while [ 1 ]; do - local line=`flock -x $lock_file -c "head -n1 $task_file;sed -i \"1d\" $task_file"` + while true; do + local line + line=$(flock -x "$lock_file" -c "head -n1 $task_file;sed -i \"1d\" $task_file") if [ "x$line" = "x%%FINISHED%%" ]; then # echo "$index . $thread_no EXIT" break @@ -156,15 +156,17 @@ function run_thread() { if [ -z "$line" ]; then continue fi - echo "$line"|grep -q "^#" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "^#"; then continue fi - local case_redo_time=`echo "$line"|cut -d, -f2` + local case_redo_time + case_redo_time=$(echo "$line" | cut -d, -f2) if [ -z "$case_redo_time" ]; then case_redo_time=${DEFAULT_RETRY_TIME:-2} fi - local case_build_san=`echo "$line"|cut -d, -f3` + local case_build_san + case_build_san=$(echo "$line" | cut -d, -f3) if [ "${case_build_san}" == "y" ]; then case_build_san="y" DEBUGPATH="debugSan" @@ -175,140 +177,150 @@ function run_thread() { usage exit 1 fi - local exec_dir=`echo "$line"|cut -d, -f4` - local case_cmd=`echo "$line"|cut -d, -f5` + local exec_dir + exec_dir=$(echo "$line" | cut -d, -f4) + local case_cmd + case_cmd=$(echo "$line" | cut -d, -f5) local case_file="" - echo "$case_cmd"|grep -q "\.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sh"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sh" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^python3" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^python3"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "^./pytest.sh" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "^./pytest.sh"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.py" | awk '{print $NF}') fi - echo "$case_cmd"|grep -q "\.sim" - if [ $? -eq 0 ]; then - case_file=`echo "$case_cmd"|grep -o ".*\.sim"|awk '{print $NF}'` + + if echo "$case_cmd" | grep -q "\.sim"; then + case_file=$(echo "$case_cmd" | grep -o ".*\.sim" | awk '{print $NF}') fi if [ -z "$case_file" ]; then - case_file=`echo "$case_cmd"|awk '{print $NF}'` + case_file=$(echo "$case_cmd" | awk '{print $NF}') fi if [ -z "$case_file" ]; then continue fi case_sql_file="$exec_dir/${case_file}.sql" case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" - count=$(( count + 1 )) - local case_path=`dirname "$case_file"` - if [ ! -z "$case_path" ]; then - mkdir -p $log_dir/$case_path + count=$((count + 1)) + local case_path + case_path=$(dirname "$case_file") + if [ -n "$case_path" ]; then + mkdir -p "$log_dir"/"$case_path" fi cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} -s ${case_build_san} ${timeout_param}" # echo "$thread_no $count $cmd" local ret=0 local redo_count=1 local case_log_file=$log_dir/${case_file}.txt - start_time=`date +%s` - local case_index=`flock -x $lock_file -c "sh -c \"echo \\\$(( \\\$( cat $index_file ) + 1 )) | tee $index_file\""` - case_index=`printf "%5d" $case_index` - local case_info=`echo "$line"|cut -d, -f 3,4,5` + start_time=$(date +%s) + local case_index + case_index=$(flock -x "$lock_file" -c "sh -c \"echo \$(( \$( cat $index_file ) + 1 )) | tee $index_file\"") + case_index=$(printf "%5d" "$case_index") + local case_info + case_info=$(echo "$line" | cut -d, -f 3,4,5) while [ ${redo_count} -lt 6 ]; do - if [ -f $case_log_file ]; then - cp $case_log_file $log_dir/$case_file.${redo_count}.redotxt + if [ -f "$case_log_file" ]; then + cp "$case_log_file" "$log_dir"/"$case_file".${redo_count}.redotxt fi - echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$case_log_file - local current_time=`date "+%Y-%m-%d %H:%M:%S"` + echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >"$case_log_file" + local current_time + current_time=$(date "+%Y-%m-%d %H:%M:%S") echo -e "$case_index \e[33m START >>>>> \e[0m ${case_info} \e[33m[$current_time]\e[0m" - echo "$current_time" >>$case_log_file - local real_start_time=`date +%s` + echo "$current_time" >>"$case_log_file" + local real_start_time + real_start_time=$(date +%s) # $cmd 2>&1 | tee -a $case_log_file # ret=${PIPESTATUS[0]} - $cmd >>$case_log_file 2>&1 + $cmd >>"$case_log_file" 2>&1 ret=$? - local real_end_time=`date +%s` - local time_elapsed=$(( real_end_time - real_start_time )) - echo "execute time: ${time_elapsed}s" >>$case_log_file - current_time=`date "+%Y-%m-%d %H:%M:%S"` - echo "${hosts[index]} $current_time exit code:${ret}" >>$case_log_file + local real_end_time + real_end_time=$(date +%s) + local time_elapsed + time_elapsed=$((real_end_time - real_start_time)) + echo "execute time: ${time_elapsed}s" >>"$case_log_file" + current_time=$(date "+%Y-%m-%d %H:%M:%S") + echo "${hosts[index]} $current_time exit code:${ret}" >>"$case_log_file" if [ $ret -eq 0 ]; then break fi redo=0 - grep -q "wait too long for taosd start" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "wait too long for taosd start" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "ssh_exchange_identification: Connection closed by remote host" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "ssh_exchange_identification: Connection closed by remote host" "$case_log_file"; then redo=1 fi - grep -q "kex_exchange_identification: read: Connection reset by peer" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "kex_exchange_identification: read: Connection reset by peer" "$case_log_file"; then redo=1 fi - grep -q "Database not ready" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Database not ready" "$case_log_file"; then redo=1 fi - grep -q "Unable to establish connection" $case_log_file - if [ $? -eq 0 ]; then + + if grep -q "Unable to establish connection" "$case_log_file"; then redo=1 fi - if [ $redo_count -lt $case_redo_time ]; then + if [ $redo_count -lt "$case_redo_time" ]; then redo=1 fi if [ $redo -eq 0 ]; then break fi - redo_count=$(( redo_count + 1 )) + redo_count=$((redo_count + 1)) done - end_time=`date +%s` - echo >>$case_log_file - total_time=$(( end_time - start_time )) - echo "${hosts[index]} total time: ${total_time}s" >>$case_log_file + end_time=$(date +%s) + echo >>"$case_log_file" + total_time=$((end_time - start_time)) + echo "${hosts[index]} total time: ${total_time}s" >>"$case_log_file" # echo "$thread_no ${line} DONE" if [ $ret -eq 0 ]; then echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[32m success\e[0m" - flock -x $lock_file -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" + flock -x "$lock_file" -c "echo \"${case_info}|success|${total_time}\" >>${success_case_file}" else - if [ ! -z ${web_server} ]; then - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" + if [ -n "${web_server}" ]; then + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n ${web_server}/$test_log_dir/${case_file}.txt\" >>${failed_case_file}" else - flock -x $lock_file -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" + flock -x "$lock_file" -c "echo -e \"${hosts[index]} ret:${ret} ${line}\n log file: ${case_log_file}\" >>${failed_case_file}" fi - mkdir -p $log_dir/${case_file}.coredump + mkdir -p "${log_dir}"/"${case_file}".coredump local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump" local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" $cmd # 2>/dev/null - local corefile=`ls $log_dir/${case_file}.coredump/` + local corefile + corefile=$(ls "$log_dir/${case_file}.coredump/") echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m" echo "=========================log============================" - cat $case_log_file + cat "$case_log_file" echo "=====================================================" echo -e "\e[34m log file: $case_log_file \e[0m" - if [ ! -z "${web_server}" ]; then + if [ -n "${web_server}" ]; then echo "${web_server}/$test_log_dir/${case_file}.txt" fi - if [ ! -z "$corefile" ]; then + if [ -n "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" # if [ $ent -ne 0 ]; then # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" # fi - mkdir $build_dir 2>/dev/null + mkdir "$build_dir" 2>/dev/null if [ $? -eq 0 ]; then # scp build binary cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" @@ -319,15 +331,15 @@ function run_thread() { # get remote sim dir local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" local tarcmd="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then tarcmd="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" fi cmd="$tarcmd sh -c \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" $cmd local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" - local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file)" + local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file}" scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" - if [ -z ${passwords[index]} ]; then + if [ -z "${passwords[index]}" ]; then scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" fi cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" @@ -341,11 +353,9 @@ function run_thread() { source_tar_dir=$log_dir/TDinternal_${hosts[index]} source_tar_file=TDinternal.tar.gz fi - mkdir $source_tar_dir 2>/dev/null + mkdir "$source_tar_dir" 2>/dev/null if [ $? -eq 0 ]; then cmd="$scpcmd:${workdirs[index]}/$source_tar_file $source_tar_dir" - # echo "$cmd" - # $cmd fi fi done @@ -361,12 +371,12 @@ function run_thread() { i=0 while [ $i -lt ${#hosts[*]} ]; do clean_tmp $i & - i=$(( i + 1 )) + i=$((i + 1)) done wait -mkdir -p $log_dir -rm -rf $log_dir/* +mkdir -p "$log_dir" +rm -rf "${log_dir:?}"/* task_file=$log_dir/$$.task lock_file=$log_dir/$$.lock index_file=$log_dir/case_index.txt @@ -374,71 +384,71 @@ stat_file=$log_dir/stat.txt failed_case_file=$log_dir/failed.txt success_case_file=$log_dir/success.txt -echo "0" >$index_file +echo "0" >"$index_file" i=0 j=0 while [ $i -lt ${#hosts[*]} ]; do - j=$(( j + threads[i] )) - i=$(( i + 1 )) + j=$((j + threads[i])) + i=$((i + 1)) done prepare_cases $j i=0 while [ $i -lt ${#hosts[*]} ]; do j=0 - while [ $j -lt ${threads[i]} ]; do + while [ $j -lt "${threads[i]}" ]; do run_thread $i $j & - j=$(( j + 1 )) + j=$((j + 1)) done - i=$(( i + 1 )) + i=$((i + 1)) done wait -rm -f $lock_file -rm -f $task_file +rm -f "$lock_file" +rm -f "$task_file" # docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f echo "=====================================================================" echo "log dir: $log_dir" -total_cases=`cat $index_file` +total_cases=$(cat "$index_file") failed_cases=0 -if [ -f $failed_case_file ]; then - if [ ! -z "$web_server" ]; then - failed_cases=`grep -v "$web_server" $failed_case_file|wc -l` +if [ -f "$failed_case_file" ]; then + if [ -n "$web_server" ]; then + failed_cases=$(grep -c -v "$web_server" "$failed_case_file") else - failed_cases=`grep -v "log file:" $failed_case_file|wc -l` + failed_cases=$(grep -c -v "log file:" "$failed_case_file") fi fi -success_cases=$(( total_cases - failed_cases )) -echo "Total Cases: $total_cases" >$stat_file -echo "Successful: $success_cases" >>$stat_file -echo "Failed: $failed_cases" >>$stat_file -cat $stat_file +success_cases=$((total_cases - failed_cases)) +echo "Total Cases: $total_cases" >"$stat_file" +echo "Successful: $success_cases" >>"$stat_file" +echo "Failed: $failed_cases" >>"$stat_file" +cat "$stat_file" RET=0 i=1 if [ -f "${failed_case_file}" ]; then echo "=====================================================" - while read line; do - if [ ! -z "${web_server}" ]; then - echo "$line"|grep -q "${web_server}" - if [ $? -eq 0 ]; then + while read -r line; do + if [ -n "${web_server}" ]; then + + if echo "$line" | grep -q "${web_server}"; then echo " $line" continue fi else - echo "$line"|grep -q "log file:" - if [ $? -eq 0 ]; then + + if echo "$line" | grep -q "log file:"; then echo " $line" continue fi fi - line=`echo "$line"|cut -d, -f 3,4,5` + line=$(echo "$line" | cut -d, -f 3,4,5) echo -e "$i. $line \e[31m failed\e[0m" >&2 - i=$(( i + 1 )) - done <${failed_case_file} + i=$((i + 1)) + done <"${failed_case_file}" RET=1 fi diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index bae0fdf278..2837c817be 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -60,16 +60,16 @@ echo "ASAN_DIR : $ASAN_DIR" # prevent delete / folder or /usr/bin if [ ${#SIM_DIR} -lt 10 ]; then - echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" - exit 1 + echo "len(SIM_DIR) < 10 , danger so exit. SIM_DIR=$SIM_DIR" + exit 1 fi -rm -rf $SIM_DIR/* +rm -rf "${SIM_DIR:?}"/* mkdir -p $PRG_DIR mkdir -p $ASAN_DIR -cd $CODE_DIR +cd "$CODE_DIR" || exit ulimit -n 600000 ulimit -c unlimited @@ -78,10 +78,10 @@ ulimit -c unlimited echo "ExcuteCmd:" $* if [[ "$TD_OS" == "Alpine" ]]; then - $* + "$@" else AsanFile=$ASAN_DIR/psim.info - echo "AsanFile:" $AsanFile + echo "AsanFile:" "$AsanFile" unset LD_PRELOAD #export LD_PRELOAD=libasan.so.5