use docker container to run cases

Jenkinsfile2

@@ -73,23 +73,11 @@ def pre_test(){
         git pull >/dev/null
         git fetch origin +refs/pull/${CHANGE_ID}/merge
         git checkout -qf FETCH_HEAD
-        git submodule update --init --recursive
-    '''
-    sh '''
-        cd ${WKC}
-        export TZ=Asia/Harbin
-        date
-        rm -rf debug
-        mkdir debug
-        cd debug
-        cmake .. > /dev/null
-        make -j4> /dev/null
     '''
     sh '''
         cd ${WKPY}
         git reset --hard
         git pull
-        pip3 install .
     '''
     return 1
 }
@@ -161,9 +149,9 @@ pipeline {
     agent none
     options { skipDefaultCheckout() } 
     environment{
-        WK = '/var/lib/jenkins/workspace/TDinternal'
+        WKDIR = '/var/lib/jenkins/workspace/v3.0'
-        WKC= '/var/lib/jenkins/workspace/TDengine'
+        WKC= '/var/lib/jenkins/workspace/v3.0/TDengine'
-        WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
+        WKPY= '/var/lib/jenkins/workspace/v3.0/taos-connector-python'
     }
     stages {
         stage('run test') {
@@ -182,20 +170,13 @@ pipeline {
                         changeRequest()
                     }
                     steps {
-                        timeout(time: 45, unit: 'MINUTES'){
+                        timeout(time: 20, unit: 'MINUTES'){
                             pre_test()
                             sh '''
-                                cd ${WKC}/debug
+                                cd ${WKC}/tests/parallel_test
-                                ctest -VV
+                                time ./container_build.sh -w ${WKDIR} -t 8
-                            '''
+                                ./collect_cases.sh
-                            sh '''
+                                time ./run.sh -m /home/m.json -t /tmp/cases.task -b 3.0 -l ${WKDIR}/log
-                                export LD_LIBRARY_PATH=${WKC}/debug/build/lib
-                                cd ${WKC}/tests/system-test
-                                ./fulltest.sh
-                            '''
-                            sh '''
-                                cd ${WKC}/tests
-                                ./test-all.sh b1fq
                             '''
                         }
                     }

tests/parallel_test/collect_cases.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+case_file=/tmp/cases.task
+
+if [ ! -z $1 ]; then
+    case_file="$1"
+fi
+
+script_dir=`dirname $0`
+cd $script_dir
+
+echo ",,unit-test,bash test.sh" >$case_file
+cat ../script/jenkins/basic.txt |grep -v "^#"|grep -v "^$"|sed "s/^/,,script,/" >>$case_file
+grep "^python" ../system-test/fulltest.sh |sed "s/^/,,system-test,/" >>$case_file
+
+exit 0
+

tests/parallel_test/container_build.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -e enterprise edition"
+    echo -e "\t -t make thread count"
+    echo -e "\t -h help"
+}
+
+ent=0
+while getopts "w:t:eh" opt; do
+    case $opt in
+        w)
+            WORKDIR=$OPTARG
+            ;;
+        e)
+            ent=1
+            ;;
+        t)
+            THREAD_COUNT=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORKDIR" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$THREAD_COUNT" ]; then
+    THREAD_COUNT=1
+fi
+
+ulimit -c unlimited
+
+if [ $ent -eq 0 ]; then
+    REP_DIR=/home/TDengine
+    REP_MOUNT_PARAM=$WORKDIR/TDengine:/home/TDengine
+else
+    REP_DIR=/home/TDinternal
+    REP_MOUNT_PARAM=$WORKDIR/TDinternal:/home/TDinternal
+fi
+
+docker run \
+    -v $REP_MOUNT_PARAM \
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake ..;make -j $THREAD_COUNT"
+
+ret=$?
+exit $ret
+

tests/parallel_test/run.sh

@@ -0,0 +1,342 @@
+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -m vm config file"
+    echo -e "\t -t task file"
+    echo -e "\t -b branch"
+    echo -e "\t -l log dir"
+    echo -e "\t -e enterprise edition"
+    echo -e "\t -o default timeout value"
+    echo -e "\t -h help"
+}
+
+ent=0
+while getopts "m:t:b:l:o:eh" opt; do
+    case $opt in
+        m)
+            config_file=$OPTARG
+            ;;
+        t)
+            t_file=$OPTARG
+            ;;
+        b)
+            branch=$OPTARG
+            ;;
+        l)
+            log_dir=$OPTARG
+            ;;
+        e)
+            ent=1
+            ;;
+        o)
+            timeout_param="-o $OPTARG"
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+#config_file=$1
+if [ -z $config_file ]; then
+    usage
+    exit 1
+fi
+if [ ! -f $config_file ]; then
+    echo "$config_file not found"
+    usage
+    exit 1
+fi
+#t_file=$2
+if [ -z $t_file ]; then
+    usage
+    exit 1
+fi
+if [ ! -f $t_file ]; then
+    echo "$t_file not found"
+    usage
+    exit 1
+fi
+date_tag=`date +%Y%m%d-%H%M%S`
+if [ -z $log_dir ]; then
+    log_dir="log/${branch}_${date_tag}"
+else
+    log_dir="$log_dir/${branch}_${date_tag}"
+fi
+
+hosts=()
+usernames=()
+passwords=()
+workdirs=()
+threads=()
+
+i=0
+while [ 1 ]; do
+    host=`jq .[$i].host $config_file`
+    if [ "$host" = "null" ]; then
+        break
+    fi
+    username=`jq .[$i].username $config_file`
+    if [ "$username" = "null" ]; then
+        break
+    fi
+    password=`jq .[$i].password $config_file`
+    if [ "$password" = "null" ]; then
+        password=""
+    fi
+    workdir=`jq .[$i].workdir $config_file`
+    if [ "$workdir" = "null" ]; then
+        break
+    fi
+    thread=`jq .[$i].thread $config_file`
+    if [ "$thread" = "null" ]; then
+        break
+    fi
+    hosts[i]=`echo $host|sed 's/\"$//'|sed 's/^\"//'`
+    usernames[i]=`echo $username|sed 's/\"$//'|sed 's/^\"//'`
+    passwords[i]=`echo $password|sed 's/\"$//'|sed 's/^\"//'`
+    workdirs[i]=`echo $workdir|sed 's/\"$//'|sed 's/^\"//'`
+    threads[i]=$thread
+    i=$(( i + 1 ))
+done
+
+
+function prepare_cases() {
+    cat $t_file >>$task_file
+    local i=0
+    while [ $i -lt $1 ]; do
+        echo "%%FINISHED%%" >>$task_file
+        i=$(( i + 1 ))
+    done
+}
+
+function clean_tmp() {
+    # clean tmp dir
+    local index=$1
+    local ssh_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}"
+    if [ -z ${passwords[index]} ]; then
+        ssh_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}"
+    fi
+    local cmd="${ssh_script} rm -rf ${workdirs[index]}/tmp"
+    ${cmd}
+}
+
+function run_thread() {
+    local index=$1
+    local thread_no=$2
+    local runcase_script="sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}"
+    if [ -z ${passwords[index]} ]; then
+        runcase_script="ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}"
+    fi
+    local count=0
+    local script="${workdirs[index]}/TDengine/tests/parallel_test/run_container.sh"
+    if [ $ent -ne 0 ]; then
+        local script="${workdirs[index]}/TDinternal/community/tests/parallel_test/run_container.sh"
+    fi
+    local cmd="${runcase_script} ${script}"
+
+    # script="echo"
+    while [ 1 ]; do
+        local line=`flock -x $lock_file -c "head -n1 $task_file;sed -i \"1d\" $task_file"`
+        if [ "x$line" = "x%%FINISHED%%" ]; then
+            # echo "$index . $thread_no EXIT"
+            break
+        fi
+        if [ -z "$line" ]; then
+            continue
+        fi
+        echo "$line"|grep -q "^#"
+        if [ $? -eq 0 ]; then
+            continue
+        fi
+        local case_redo_time=`echo "$line"|cut -d, -f2`
+        if [ -z "$case_redo_time" ]; then
+            case_redo_time=${DEFAULT_RETRY_TIME:-2}
+        fi
+        local exec_dir=`echo "$line"|cut -d, -f3`
+        local case_cmd=`echo "$line"|cut -d, -f4`
+        local case_file=""
+        echo "$case_cmd"|grep -q "\.sh"
+        if [ $? -eq 0 ]; then
+            case_file=`echo "$case_cmd"|grep -o ".*\.sh"|awk '{print $NF}'`
+        fi
+        echo "$case_cmd"|grep -q "^python3"
+        if [ $? -eq 0 ]; then
+            case_file=`echo "$case_cmd"|grep -o ".*\.py"|awk '{print $NF}'`
+        fi
+        echo "$case_cmd"|grep -q "\.sim"
+        if [ $? -eq 0 ]; then
+            case_file=`echo "$case_cmd"|grep -o ".*\.sim"|awk '{print $NF}'`
+        fi
+        if [ -z "$case_file" ]; then
+            case_file=`echo "$case_cmd"|awk '{print $NF}'`
+        fi
+        if [ -z "$case_file" ]; then
+            continue
+        fi
+        case_file="$exec_dir/${case_file}.${index}.${thread_no}.${count}"
+        count=$(( count + 1 ))
+        local case_path=`dirname "$case_file"`
+        if [ ! -z "$case_path" ]; then
+            mkdir -p $log_dir/$case_path
+        fi
+        cmd="${runcase_script} ${script} -w ${workdirs[index]} -c \"${case_cmd}\" -t ${thread_no} -d ${exec_dir} ${timeout_param}"
+        # echo "$thread_no $count $cmd"
+        local ret=0
+        local redo_count=1
+        start_time=`date +%s`
+        while [ ${redo_count} -lt 6 ]; do
+            if [ -f $log_dir/$case_file.log ]; then
+                cp $log_dir/$case_file.log $log_dir/$case_file.${redo_count}.redolog
+            fi
+            echo "${hosts[index]}-${thread_no} order:${count}, redo:${redo_count} task:${line}" >$log_dir/$case_file.log
+            echo -e "\e[33m >>>>> \e[0m ${case_cmd}"
+            date >>$log_dir/$case_file.log
+            # $cmd 2>&1 | tee -a $log_dir/$case_file.log
+            # ret=${PIPESTATUS[0]}
+            $cmd >>$log_dir/$case_file.log 2>&1
+            ret=$?
+            echo "${hosts[index]} `date` ret:${ret}" >>$log_dir/$case_file.log
+            if [ $ret -eq 0 ]; then
+                break
+            fi
+            redo=0
+            grep -q "wait too long for taosd start" $log_dir/$case_file.log
+            if [ $? -eq 0 ]; then
+                redo=1
+            fi
+            grep -q "kex_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log
+            if [ $? -eq 0 ]; then
+                redo=1
+            fi
+            grep -q "ssh_exchange_identification: Connection closed by remote host" $log_dir/$case_file.log
+            if [ $? -eq 0 ]; then
+                redo=1
+            fi
+            grep -q "kex_exchange_identification: read: Connection reset by peer" $log_dir/$case_file.log
+            if [ $? -eq 0 ]; then
+                redo=1
+            fi
+            grep -q "Database not ready" $log_dir/$case_file.log
+            if [ $? -eq 0 ]; then
+                redo=1
+            fi
+            grep -q "Unable to establish connection" $log_dir/$case_file.log
+            if [ $? -eq 0 ]; then
+                redo=1
+            fi
+            if [ $redo_count -lt $case_redo_time ]; then
+                redo=1
+            fi
+            if [ $redo -eq 0 ]; then
+                break
+            fi
+            redo_count=$(( redo_count + 1 ))
+        done
+        end_time=`date +%s`
+        echo >>$log_dir/$case_file.log
+        echo "${hosts[index]} execute time: $(( end_time - start_time ))s" >>$log_dir/$case_file.log
+        # echo "$thread_no ${line} DONE"
+        if [ $ret -ne 0 ]; then
+            flock -x $lock_file -c "echo \"${hosts[index]} ret:${ret} ${line}\" >>$log_dir/failed.log"
+            mkdir -p $log_dir/${case_file}.coredump
+            local remote_coredump_dir="${workdirs[index]}/tmp/thread_volume/$thread_no/coredump"
+            local scpcmd="sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}"
+            if [ -z ${passwords[index]} ]; then
+                scpcmd="scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}"
+            fi
+            cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/"
+            $cmd # 2>/dev/null
+            local case_info=`echo "$line"|cut -d, -f 3,4`
+            local corefile=`ls $log_dir/${case_file}.coredump/`
+            corefile=`find $log_dir/${case_file}.coredump/ -name "core.*"`
+            echo -e "$case_info \e[31m failed\e[0m"
+            echo "=========================log============================"
+            cat $log_dir/$case_file.log
+            echo "====================================================="
+            echo -e "\e[34m log file: $log_dir/$case_file.log \e[0m"
+            if [ ! -z "$corefile" ]; then
+                echo -e "\e[34m corefiles: $corefile \e[0m"
+                local build_dir=$log_dir/build_${hosts[index]}
+                local remote_build_dir="${workdirs[index]}/TDengine/debug/build"
+                if [ $ent -ne 0 ]; then
+                    remote_build_dir="${workdirs[index]}/TDinternal/debug/build"
+                fi
+                mkdir $build_dir 2>/dev/null
+                if [ $? -eq 0 ]; then
+                    # scp build binary
+                    cmd="$scpcmd:${remote_build_dir}/* ${build_dir}/"
+                    echo "$cmd"
+                    $cmd >/dev/null
+                fi
+            fi
+        fi
+    done
+}
+
+# echo "hosts: ${hosts[@]}"
+# echo "usernames: ${usernames[@]}"
+# echo "passwords: ${passwords[@]}"
+# echo "workdirs: ${workdirs[@]}"
+# echo "threads: ${threads[@]}"
+# TODO: check host accessibility
+
+i=0
+while [ $i -lt ${#hosts[*]} ]; do
+    clean_tmp $i &
+    i=$(( i + 1 ))
+done
+wait
+
+mkdir -p $log_dir
+rm -rf $log_dir/*
+task_file=$log_dir/$$.task
+lock_file=$log_dir/$$.lock
+
+i=0
+j=0
+while [ $i -lt ${#hosts[*]} ]; do
+    j=$(( j + threads[i] ))
+    i=$(( i + 1 ))
+done
+prepare_cases $j
+
+i=0
+while [ $i -lt ${#hosts[*]} ]; do
+    j=0
+    while [ $j -lt ${threads[i]} ]; do
+        run_thread $i $j &
+        j=$(( j + 1 ))
+    done
+    i=$(( i + 1 ))
+done
+
+wait
+
+rm -f $lock_file
+rm -f $task_file
+
+# docker ps -a|grep -v CONTAINER|awk '{print $1}'|xargs docker rm -f
+RET=0
+i=1
+if [ -f "$log_dir/failed.log" ]; then
+    echo "====================================================="
+    while read line; do
+        line=`echo "$line"|cut -d, -f 3,4`
+        echo -e "$i. $line \e[31m failed\e[0m" >&2
+        i=$(( i + 1 ))
+    done <$log_dir/failed.log
+    RET=1
+fi
+
+echo "${log_dir}" >&2
+
+date
+
+exit $RET

tests/parallel_test/run_case.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -d execution dir"
+    echo -e "\t -c command"
+    echo -e "\t -e enterprise edition"
+    echo -e "\t -o default timeout value"
+    echo -e "\t -h help"
+}
+
+ent=0
+while getopts "d:c:o:eh" opt; do
+    case $opt in
+        d)
+            exec_dir=$OPTARG
+            ;;
+        c)
+            cmd=$OPTARG
+            ;;
+        o)
+            TIMEOUT_CMD="timeout $OPTARG"
+            ;;
+        e)
+            ent=1
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$exec_dir" ]; then
+    usage
+    exit 0
+fi
+if [ -z "$cmd" ]; then
+    usage
+    exit 0
+fi
+
+if [ $ent -eq 0 ]; then
+    export PATH=$PATH:/home/TDengine/debug/build/bin
+    export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib
+    ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
+    CONTAINER_TESTDIR=/home/TDengine
+else
+    export PATH=$PATH:/home/TDinternal/debug/build/bin
+    export LD_LIBRARY_PATH=/home/TDinternal/debug/build/lib
+    ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null
+    CONTAINER_TESTDIR=/home/TDinternal/community
+fi
+mkdir -p /var/lib/taos/subscribe
+mkdir -p /var/log/taos
+mkdir -p /var/lib/taos
+
+cd $CONTAINER_TESTDIR/tests/$exec_dir
+ulimit -c unlimited
+
+$TIMEOUT_CMD $cmd
+RET=$?
+
+if [ $RET -ne 0 ]; then
+    pwd
+fi
+
+exit $RET
+

tests/parallel_test/run_container.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -d execution dir"
+    echo -e "\t -c command"
+    echo -e "\t -t thread number"
+    echo -e "\t -e enterprise edition"
+    echo -e "\t -o default timeout value"
+    echo -e "\t -h help"
+}
+
+ent=0
+while getopts "w:d:c:t:o:eh" opt; do
+    case $opt in
+        w)
+            WORKDIR=$OPTARG
+            ;;
+        d)
+            exec_dir=$OPTARG
+            ;;
+        c)
+            cmd=$OPTARG
+            ;;
+        t)
+            thread_no=$OPTARG
+            ;;
+        e)
+            ent=1
+            ;;
+        o)
+            extra_param="-o $OPTARG"
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORKDIR" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$exec_dir" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$cmd" ]; then
+    usage
+    exit 1
+fi
+if [ -z "$thread_no" ]; then
+    usage
+    exit 1
+fi
+if [ $ent -ne 0 ]; then
+    # enterprise edition
+    extra_param="$extra_param -e"
+    INTERNAL_REPDIR=$WORKDIR/TDinternal
+    REPDIR=$INTERNAL_REPDIR/community
+    CONTAINER_TESTDIR=/home/TDinternal/community
+    REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal"
+else
+    # community edition
+    REPDIR=$WORKDIR/TDengine
+    CONTAINER_TESTDIR=/home/TDengine
+    REP_MOUNT_PARAM="$REPDIR:/home/TDengine"
+fi
+
+ulimit -c unlimited
+
+TMP_DIR=$WORKDIR/tmp
+
+MOUNT_DIR=""
+mkdir -p ${TMP_DIR}/thread_volume/$thread_no/sim/tsim
+mkdir -p ${TMP_DIR}/thread_volume/$thread_no/coredump
+rm -rf ${TMP_DIR}/thread_volume/$thread_no/coredump/*
+if [ ! -d "${TMP_DIR}/thread_volume/$thread_no/$exec_dir" ]; then
+    subdir=`echo "$exec_dir"|cut -d/ -f1`
+    echo "cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/"
+    cp -rf ${REPDIR}/tests/$subdir ${TMP_DIR}/thread_volume/$thread_no/
+fi
+MOUNT_DIR="$TMP_DIR/thread_volume/$thread_no/$exec_dir:$CONTAINER_TESTDIR/tests/$exec_dir"
+echo "$thread_no -> ${exec_dir}:$cmd"
+coredump_dir=`cat /proc/sys/kernel/core_pattern | xargs dirname`
+
+docker run \
+    -v $REP_MOUNT_PARAM \
+    -v $MOUNT_DIR \
+    -v "$TMP_DIR/thread_volume/$thread_no/sim:${CONTAINER_TESTDIR}/sim" \
+    -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \
+    -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \
+    --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd" $extra_param
+ret=$?
+exit $ret
+

tests/unit-test/test.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -e enterprise edition"
+    echo -e "\t -h help"
+}
+
+ent=0
+while getopts "eh" opt; do
+    case $opt in
+        e)
+            ent=1
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+script_dir=`dirname $0`
+cd ${script_dir}
+PWD=`pwd`
+
+if [ $ent -eq 0 ]; then
+    cd ../../debug
+else
+    cd ../../../debug
+fi
+
+ctest -j8
+ret=$?
+exit $ret
+