diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 96e29328fa..dacb0a8702 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -73,23 +73,11 @@ def pre_test(){ git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git submodule update --init --recursive - ''' - sh ''' - cd ${WKC} - export TZ=Asia/Harbin - date - rm -rf debug - mkdir debug - cd debug - cmake .. > /dev/null - make -j4> /dev/null ''' sh ''' cd ${WKPY} git reset --hard git pull - pip3 install . ''' return 1 } @@ -161,9 +149,9 @@ pipeline { agent none options { skipDefaultCheckout() } environment{ - WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' + WKDIR = '/var/lib/jenkins/workspace/v3.0' + WKC= '/var/lib/jenkins/workspace/v3.0/TDengine' + WKPY= '/var/lib/jenkins/workspace/v3.0/taos-connector-python' } stages { stage('run test') { @@ -182,20 +170,13 @@ pipeline { changeRequest() } steps { - timeout(time: 45, unit: 'MINUTES'){ + timeout(time: 20, unit: 'MINUTES'){ pre_test() sh ''' - cd ${WKC}/debug - ctest -VV - ''' - sh ''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - sh ''' - cd ${WKC}/tests - ./test-all.sh b1fq + cd ${WKC}/tests/parallel_test + time ./container_build.sh -w ${WKDIR} -t 8 + ./collect_cases.sh + time ./run.sh -m /home/m.json -t /tmp/cases.task -b 3.0 -l ${WKDIR}/log ''' } } diff --git a/tests/parallel_test/collect_cases.sh b/tests/parallel_test/collect_cases.sh new file mode 100755 index 0000000000..a513d62d94 --- /dev/null +++ b/tests/parallel_test/collect_cases.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +case_file=/tmp/cases.task + +if [ ! -z $1 ]; then + case_file="$1" +fi + +script_dir=`dirname $0` +cd $script_dir + +echo ",,unit-test,bash test.sh" >$case_file +cat ../script/jenkins/basic.txt |grep -v "^#"|grep -v "^$"|sed "s/^/,,script,/" >>$case_file +grep "^python" ../system-test/fulltest.sh |sed "s/^/,,system-test,/" >>$case_file + +exit 0 + diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh new file mode 100755 index 0000000000..c1571bf279 --- /dev/null +++ b/tests/parallel_test/container_build.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -w work dir" + echo -e "\t -e enterprise edition" + echo -e "\t -t make thread count" + echo -e "\t -h help" +} + +ent=0 +while getopts "w:t:eh" opt; do + case $opt in + w) + WORKDIR=$OPTARG + ;; + e) + ent=1 + ;; + t) + THREAD_COUNT=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi +if [ -z "$THREAD_COUNT" ]; then + THREAD_COUNT=1 +fi + +ulimit -c unlimited + +if [ $ent -eq 0 ]; then + REP_DIR=/home/TDengine + REP_MOUNT_PARAM=$WORKDIR/TDengine:/home/TDengine +else + REP_DIR=/home/TDinternal + REP_MOUNT_PARAM=$WORKDIR/TDinternal:/home/TDinternal +fi + +docker run \ + -v $REP_MOUNT_PARAM \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake ..;make -j $THREAD_COUNT" + +ret=$? +exit $ret + diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh new file mode 100755 index 0000000000..a2d3da1564 --- /dev/null +++ b/tests/parallel_test/run.sh @@ -0,0 +1,342 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -m vm config file" + echo -e "\t -t task file" + echo -e "\t -b branch" + echo -e "\t -l log dir" + echo -e "\t -e enterprise edition" + echo -e "\t -o default timeout value" + echo -e "\t -h help" +} + +ent=0 +while getopts "m:t:b:l:o:eh" opt; do + case $opt in + m) + config_file=$OPTARG + ;; + t) + t_file=$OPTARG + ;; + b) + branch=$OPTARG + ;; + l) + log_dir=$OPTARG + ;; + e) + ent=1 + ;; + o) + timeout_param="-o $OPTARG" + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done +#config_file=$1 +if [ -z $config_file ]; then + usage + exit 1 +fi +if [ ! -f $config_file ]; then + echo "$config_file not found" + usage + exit 1 +fi +#t_file=$2 +if [ -z $t_file ]; then + usage + exit 1 +fi +if [ ! -f $t_file ]; then + echo "$t_file not found" + usage + exit 1 +fi +date_tag=`date +%Y%m%d-%H%M%S` +if [ -z $log_dir ]; then + log_dir="log/${branch}_${date_tag}" +else + log_dir="$log_dir/${branch}_${date_tag}" +fi + +hosts=() +usernames=() +passwords=() +workdirs=() +threads=() + +i=0 +while [ 1 ]; do + host=`jq .[$i].host $config_file` + if [ "$host" = "null" ]; then + break + fi + username=`jq .[$i].username $config_file` + if [ "$username" = "null" ]; then + break + fi + password=`jq .[$i].password $config_file` + if [ "$password" = "null" ]; then + password="" + fi + workdir=`jq .[$i].workdir $config_file` + if [ "$workdir" = "null" ]; then + break + fi + thread=`jq .[$i].thread $config_file` + if [ "$thread" = "null" ]; then + break + fi + hosts[i]=`echo $host|sed 's/\"$//'|sed 's/^\"//'` + usernames[i]=`echo $username|sed 's/\"$//'|sed 's/^\"//'` + passwords[i]=`echo $password|sed 's/\"$//'|sed 's/^\"//'` + workdirs[i]=`echo $workdir|sed 's/\"$//'|sed 's/^\"//'` + threads[i]=$thread + i=$(( i + 1 )) +done + + +function prepare_cases() { + cat $t_file >>$task_file + local i=0 + while [ $i -lt $1 ]; do + echo "%%FINISHED%%" >>$task_file + i=$(( i + 1 )) + done +} + +function clean_tmp() { + # clean tmp dir + local index=$1 + local ssh_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" + if [ -z ${passwords[index]} ]; then + ssh_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" + fi + local cmd="${ssh_script} rm -rf ${workdirs[index]}/tmp" + ${cmd} +} + +function run_thread() { + local index=$1 + local thread_no=$2 + local runcase_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" + if [ -z ${passwords[index]} ]; then + runcase_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" + fi + local count=0 + local script="${workdirs[index]}/TDengine/tests/parallel_test/run_container.sh" + if [ $ent -ne 0 ]; then + local script="${workdirs[index]}/TDinternal/community/tests/parallel_test/run_container.sh" + fi + local cmd="${runcase_script} ${script}" + + # script="echo" + while [ 1 ]; do + local line=`flock -x $lock_file -c "head -n1 $task_file;sed -i \"1d\" $task_file"` + if [ "x$line" = "x%%FINISHED%%" ]; then + # echo "$index . $thread_no EXIT" + break + fi + if [ -z "$line" ]; then + continue + fi + echo "$line"|grep -q "^#" + if [ $? -eq 0 ]; then + continue + fi + local case_redo_time=`echo "$line"|cut -d, -f2` + if [ -z "$case_redo_time" ]; then + case_redo_time=${DEFAULT_RETRY_TIME:-2} + fi + local exec_dir=`echo "$line"|cut -d, -f3` + local case_cmd=`echo "$line"|cut -d, -f4` + local case_file="" + echo "$case_cmd"|grep -q "\.sh" + if [ $? -eq 0 ]; then + case_file=`echo "$case_cmd"|grep -o ".*\.sh"|awk '{print $NF}'` + fi + echo "$case_cmd"|grep -q "^python3" + if [ $? -eq 0 ]; then + case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'` + fi + echo "$case_cmd"|grep -q "\.sim" + if [ $? -eq 0 ]; then + case_file=`echo "$case_cmd"|grep -o ".*\.sim"|awk '{print $NF}'` + fi + if [ -z "$case_file" ]; then + case_file=`echo "$case_cmd"|awk '{print $NF}'` + fi + if [ -z "$case_file" ]; then + continue + fi + case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}" + count=$(( count + 1 )) + local case_path=`dirname "$case_file"` + if [ ! -z "$case_path" ]; then + mkdir -p $log_dir/$case_path + fi + cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} ${timeout_param}" + # echo "$thread_no $count $cmd" + local ret=0 + local redo_count=1 + start_time=`date +%s` + while [ ${redo_count} -lt 6 ]; do + if [ -f $log_dir/$case_file.log ]; then + cp $log_dir/$case_file.log $log_dir/$case_file.${redo_count}.redolog + fi + echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$log_dir/$case_file.log + echo -e "\e[33m >>>>> \e[0m ${case_cmd}" + date >>$log_dir/$case_file.log + # $cmd 2>&1 | tee -a $log_dir/$case_file.log + # ret=${PIPESTATUS[0]} + $cmd >>$log_dir/$case_file.log 2>&1 + ret=$? + echo "${hosts[index]} `date` ret:${ret}" >>$log_dir/$case_file.log + if [ $ret -eq 0 ]; then + break + fi + redo=0 + grep -q "wait too long for taosd start" $log_dir/$case_file.log + if [ $? -eq 0 ]; then + redo=1 + fi + grep -q "kex_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log + if [ $? -eq 0 ]; then + redo=1 + fi + grep -q "ssh_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log + if [ $? -eq 0 ]; then + redo=1 + fi + grep -q "kex_exchange_identification: read: Connection reset by peer" $log_dir/$case_file.log + if [ $? -eq 0 ]; then + redo=1 + fi + grep -q "Database not ready" $log_dir/$case_file.log + if [ $? -eq 0 ]; then + redo=1 + fi + grep -q "Unable to establish connection" $log_dir/$case_file.log + if [ $? -eq 0 ]; then + redo=1 + fi + if [ $redo_count -lt $case_redo_time ]; then + redo=1 + fi + if [ $redo -eq 0 ]; then + break + fi + redo_count=$(( redo_count + 1 )) + done + end_time=`date +%s` + echo >>$log_dir/$case_file.log + echo "${hosts[index]} execute time: $(( end_time - start_time ))s" >>$log_dir/$case_file.log + # echo "$thread_no ${line} DONE" + if [ $ret -ne 0 ]; then + flock -x $lock_file -c "echo \"${hosts[index]} ret:${ret} ${line}\" >>$log_dir/failed.log" + mkdir -p $log_dir/${case_file}.coredump + local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump" + local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" + if [ -z ${passwords[index]} ]; then + scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" + fi + cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" + $cmd # 2>/dev/null + local case_info=`echo "$line"|cut -d, -f 3,4` + local corefile=`ls $log_dir/${case_file}.coredump/` + corefile=`find $log_dir/${case_file}.coredump/ -name "core.*"` + echo -e "$case_info \e[31m failed\e[0m" + echo "=========================log============================" + cat $log_dir/$case_file.log + echo "=====================================================" + echo -e "\e[34m log file: $log_dir/$case_file.log \e[0m" + if [ ! -z "$corefile" ]; then + echo -e "\e[34m corefiles: $corefile \e[0m" + local build_dir=$log_dir/build_${hosts[index]} + local remote_build_dir="${workdirs[index]}/TDengine/debug/build" + if [ $ent -ne 0 ]; then + remote_build_dir="${workdirs[index]}/TDinternal/debug/build" + fi + mkdir $build_dir 2>/dev/null + if [ $? -eq 0 ]; then + # scp build binary + cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/" + echo "$cmd" + $cmd >/dev/null + fi + fi + fi + done +} + +# echo "hosts: ${hosts[@]}" +# echo "usernames: ${usernames[@]}" +# echo "passwords: ${passwords[@]}" +# echo "workdirs: ${workdirs[@]}" +# echo "threads: ${threads[@]}" +# TODO: check host accessibility + +i=0 +while [ $i -lt ${#hosts[*]} ]; do + clean_tmp $i & + i=$(( i + 1 )) +done +wait + +mkdir -p $log_dir +rm -rf $log_dir/* +task_file=$log_dir/$$.task +lock_file=$log_dir/$$.lock + +i=0 +j=0 +while [ $i -lt ${#hosts[*]} ]; do + j=$(( j + threads[i] )) + i=$(( i + 1 )) +done +prepare_cases $j + +i=0 +while [ $i -lt ${#hosts[*]} ]; do + j=0 + while [ $j -lt ${threads[i]} ]; do + run_thread $i $j & + j=$(( j + 1 )) + done + i=$(( i + 1 )) +done + +wait + +rm -f $lock_file +rm -f $task_file + +# docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f +RET=0 +i=1 +if [ -f "$log_dir/failed.log" ]; then + echo "=====================================================" + while read line; do + line=`echo "$line"|cut -d, -f 3,4` + echo -e "$i. $line \e[31m failed\e[0m" >&2 + i=$(( i + 1 )) + done <$log_dir/failed.log + RET=1 +fi + +echo "${log_dir}" >&2 + +date + +exit $RET diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh new file mode 100755 index 0000000000..9705c024b8 --- /dev/null +++ b/tests/parallel_test/run_case.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -d execution dir" + echo -e "\t -c command" + echo -e "\t -e enterprise edition" + echo -e "\t -o default timeout value" + echo -e "\t -h help" +} + +ent=0 +while getopts "d:c:o:eh" opt; do + case $opt in + d) + exec_dir=$OPTARG + ;; + c) + cmd=$OPTARG + ;; + o) + TIMEOUT_CMD="timeout $OPTARG" + ;; + e) + ent=1 + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$exec_dir" ]; then + usage + exit 0 +fi +if [ -z "$cmd" ]; then + usage + exit 0 +fi + +if [ $ent -eq 0 ]; then + export PATH=$PATH:/home/TDengine/debug/build/bin + export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib + ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + CONTAINER_TESTDIR=/home/TDengine +else + export PATH=$PATH:/home/TDinternal/debug/build/bin + export LD_LIBRARY_PATH=/home/TDinternal/debug/build/lib + ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + CONTAINER_TESTDIR=/home/TDinternal/community +fi +mkdir -p /var/lib/taos/subscribe +mkdir -p /var/log/taos +mkdir -p /var/lib/taos + +cd $CONTAINER_TESTDIR/tests/$exec_dir +ulimit -c unlimited + +$TIMEOUT_CMD $cmd +RET=$? + +if [ $RET -ne 0 ]; then + pwd +fi + +exit $RET + diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh new file mode 100755 index 0000000000..b0337febcc --- /dev/null +++ b/tests/parallel_test/run_container.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -w work dir" + echo -e "\t -d execution dir" + echo -e "\t -c command" + echo -e "\t -t thread number" + echo -e "\t -e enterprise edition" + echo -e "\t -o default timeout value" + echo -e "\t -h help" +} + +ent=0 +while getopts "w:d:c:t:o:eh" opt; do + case $opt in + w) + WORKDIR=$OPTARG + ;; + d) + exec_dir=$OPTARG + ;; + c) + cmd=$OPTARG + ;; + t) + thread_no=$OPTARG + ;; + e) + ent=1 + ;; + o) + extra_param="-o $OPTARG" + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi +if [ -z "$exec_dir" ]; then + usage + exit 1 +fi +if [ -z "$cmd" ]; then + usage + exit 1 +fi +if [ -z "$thread_no" ]; then + usage + exit 1 +fi +if [ $ent -ne 0 ]; then + # enterprise edition + extra_param="$extra_param -e" + INTERNAL_REPDIR=$WORKDIR/TDinternal + REPDIR=$INTERNAL_REPDIR/community + CONTAINER_TESTDIR=/home/TDinternal/community + REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal" +else + # community edition + REPDIR=$WORKDIR/TDengine + CONTAINER_TESTDIR=/home/TDengine + REP_MOUNT_PARAM="$REPDIR:/home/TDengine" +fi + +ulimit -c unlimited + +TMP_DIR=$WORKDIR/tmp + +MOUNT_DIR="" +mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim +mkdir -p ${TMP_DIR}/thread_volume/$thread_no/coredump +rm -rf ${TMP_DIR}/thread_volume/$thread_no/coredump/* +if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then + subdir=`echo "$exec_dir"|cut -d/ -f1` + echo "cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/" + cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/ +fi +MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir" +echo "$thread_no -> ${exec_dir}:$cmd" +coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname` + +docker run \ + -v $REP_MOUNT_PARAM \ + -v $MOUNT_DIR \ + -v "$TMP_DIR/thread_volume/$thread_no/sim:${CONTAINER_TESTDIR}/sim" \ + -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \ + -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \ + --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd" $extra_param +ret=$? +exit $ret + diff --git a/tests/unit-test/test.sh b/tests/unit-test/test.sh new file mode 100755 index 0000000000..4122597717 --- /dev/null +++ b/tests/unit-test/test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +function usage() { + echo "$0" + echo -e "\t -e enterprise edition" + echo -e "\t -h help" +} + +ent=0 +while getopts "eh" opt; do + case $opt in + e) + ent=1 + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +script_dir=`dirname $0` +cd ${script_dir} +PWD=`pwd` + +if [ $ent -eq 0 ]; then + cd ../../debug +else + cd ../../../debug +fi + +ctest -j8 +ret=$? +exit $ret +