Merge pull request #26696 from taosdata/test/3.0/TD-31057

test: add the more compilation cache directories in CI
2024-07-23 17:28:41 +08:00 · 2024-07-23 17:28:41 +08:00 · d80a68512d
parent 9e36e33469 fbd17ca914
commit d80a68512d
7 changed files with 391 additions and 6 deletions
--- a/tests/ci/ci_deploy_dependency_file_list.txt
+++ b/tests/ci/ci_deploy_dependency_file_list.txt
@ -0,0 +1,8 @@
+/home/m.json
+/home/log_server.json
+/var/lib/jenkins/workspace/restore.sh
+/var/lib/jenkins/workspace/start_http.sh
+/var/lib/jenkins/workspace/TDinternal
+/var/lib/jenkins/workspace/remove_corefile.sh
+/var/lib/jenkins/workspace/CI_disk_Monitor.py
+/var/lib/jenkins/workspace/start_CI_Monitor.sh
--- a/tests/ci/container_build_newmachine.sh
+++ b/tests/ci/container_build_newmachine.sh
@ -0,0 +1,88 @@
+#!/bin/bash
+set -e 
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -e enterprise edition"
+    echo -e "\t -t make thread count"
+    echo -e "\t -h help"
+}
+
+ent=0
+while getopts "w:t:eh" opt; do
+    case $opt in
+        w)
+            WORKDIR=$OPTARG
+            ;;
+        e)
+            ent=1
+            ;;
+        t)
+            THREAD_COUNT=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORKDIR" ]; then
+    usage
+    exit 1
+fi
+# if [ -z "$THREAD_COUNT" ]; then
+#     THREAD_COUNT=1
+# fi
+
+ulimit -c unlimited
+
+if [ $ent -eq 0 ]; then
+    REP_DIR=/home/TDengine
+    REP_REAL_PATH=$WORKDIR/TDengine
+    REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine
+else
+    REP_DIR=/home/TDinternal
+    REP_REAL_PATH=$WORKDIR/TDinternal
+    REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal
+    
+fi
+date 
+docker run \
+    -v $REP_MOUNT_PARAM \
+    -v /root/.cargo/registry:/root/.cargo/registry \
+    -v /root/.cargo/git:/root/.cargo/git \
+    -v /root/go/pkg/mod:/root/go/pkg/mod \
+    -v /root/.cache/go-build:/root/.cache/go-build \
+    -v /root/.cos-local.1:/root/.cos-local.2 \
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "
+ # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
+
+if [[ -d ${WORKDIR}/debugNoSan  ]] ;then
+    echo "delete  ${WORKDIR}/debugNoSan"
+    rm -rf  ${WORKDIR}/debugNoSan
+fi
+if [[ -d ${WORKDIR}/debugSan ]] ;then
+    echo "delete  ${WORKDIR}/debugSan"
+    rm -rf  ${WORKDIR}/debugSan
+fi
+
+if [ "$(uname -m)" = "aarch64" ] ;then
+    CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Debug"
+else
+    CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Release"
+fi
+
+
+mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugSan
+date
+
+ret=$?
+exit $ret
+
--- a/tests/ci/deploy_ci_machines.sh
+++ b/tests/ci/deploy_ci_machines.sh
@ -0,0 +1,69 @@
+#!/bin/bash
+
+function scp_file_from_host {
+    # check at least three parameters
+    if [ "$#" -lt 3 ]; then
+        echo "Usage: $0 host passwd source_filename [dest_filename]"
+        exit 1
+    fi
+
+    host=$1
+    passwd=$2
+    source_filename=$3
+    # If the fourth parameter is not provided, use the third parameter as the default value
+    dest_filename=${4:-$3}  
+
+    # use sshpass and scp for secure file transfer
+    sshpass -p "$passwd" scp  -o StrictHostKeyChecking=no -r  "$host":"$source_filename" "$dest_filename"
+}
+
+
+# install docker and sshpass
+curl -fsSL https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
+sudo apt-get update
+sudo apt-get install  -y docker-ce sshpass jq 
+sudo systemctl enable docker
+sudo systemctl start docker
+
+# create a log directory
+mkdir -p /var/lib/jenkins/workspace/log
+
+# Assuming you have a file called 'file_list.txt' with one filename per line
+file_list="ci_deploy_dependency_file_list.txt"
+monitorip="192.168.1.59"
+passwd_all="abcdefg"
+
+# Read the file list and call scp_file_from_host for each file
+while IFS= read -r source_filename; do
+    scp_file_from_host "$monitorip" "$passwd_all" "$source_filename"
+done < "$file_list"
+
+# modify the configuration file
+ip=$(ifconfig |grep inet|grep 192  |awk  '{print $2}')
+sed -i "s/${monitorip}/$ip/" /home/log_server.json
+sed -i "s/${monitorip}/$ip/" /home/m.json
+
+#mkdir corefile dir and configure the system to automatically set corefile dir at startup
+mkdir -p /home/coredump/ &&  echo "echo '/home/coredump/core_%e-%p' | sudo tee /proc/sys/kernel/core_pattern " >> /root/.bashrc
+
+
+# get  image from 0.212
+image_ip="192.168.0.212"
+scp_file_from_host $image_ip $passwd_all "/home/tang/work/image/taos_image.tar " "/home/taos_image.tar"
+docker load -i /home/taos_image.tar
+
+#start http server 
+nohup /var/lib/jenkins/workspace/log/start_http.sh  &
+
+# start CI monitor and remove corefile in crontable
+(crontab -l;echo "0 1 * * * /usr/bin/bash /var/lib/jenkins/workspace/remove_corefile.sh") | crontab
+(crontab -l;echo "@reboot /usr/bin/bash  /var/lib/jenkins/workspace/start_CI_Monitor.sh") | crontab
+
+
+# generate cache dir
+cd  /var/lib/jenkins/workspace/TDinternal/community/tests/parallel_test || exit
+time ./container_build_newmachine.sh -w /var/lib/jenkins/workspace -e
+
+# test if the CI machine compilation is successful
+time ./container_build.sh -w /var/lib/jenkins/workspace -e
--- a/tests/ci/remove_corefile.sh
+++ b/tests/ci/remove_corefile.sh
@ -0,0 +1,17 @@
+#!/bin/bash
+
+date_str=$(date -d "4 day ago" +%Y%m%d)
+if [ ! -z "$1" ]; then
+    date_str="$1"
+fi
+script_dir=$(dirname $0)
+cd "${script_dir}"/log || exit
+# date >>core.list
+# find . -name "core.*" | grep "$date_str" >>core.list
+# find . -name "core.*" | grep "$date_str" | xargs rm -rf
+# find . -name "build_*" | grep "$date_str" | xargs rm -rf
+for file in *; do
+    if [[ $file == *"$date_str"* ]]; then
+        rm -rf "$file"
+    fi
+done
--- a/tests/ci/restore.sh
+++ b/tests/ci/restore.sh
@ -0,0 +1,159 @@
+#!/bin/bash
+ set -x
+JENKINS_LOG=jenkins.log
+# pr_num=14228
+# n=1
+function usage() {
+    echo "$0"
+    echo -e "\t -p PR number"
+    echo -e "\t -n build number"
+    echo -e "\t -c container name"
+    echo -e "\t -h help"
+}
+while getopts "p:n:c:h" opt; do
+    case $opt in
+        p)
+            pr_num=$OPTARG
+            ;;
+        n)
+            n=$OPTARG
+            ;;
+        c)
+            container_name=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 1
+            ;;
+    esac
+done
+if [ -z "$container_name" ]; then
+    echo "container name not specified"
+    usage
+    exit 1
+fi
+if [ -z "$pr_num" ]; then
+    echo "PR number not specified"
+    usage
+    exit 1
+fi
+if [ -z "$n" ]; then
+    echo "build number not specified"
+    usage
+    exit 1
+fi
+pr_num=`echo "$pr_num"|sed "s/PR-//"`
+container_count=`docker ps -a -f name=$container_name|wc -l`
+if [ $container_count -gt 1 ]; then
+    docker ps -a -f name=$container_name
+    echo "container $container_name exists"
+    exit 1
+fi
+cd $(dirname $0)
+info=`grep -n "^[0-9]\{8\}-[0-9]\{6\}" jenkins.log | grep -A 1 "PR-${pr_num}:${n}:"`
+# 22131:20220625-113105 NewTest/PR-14228:PR-14228:1:3.0
+# 22270:20220625-121154 NewTest/PR-14221:PR-14221:2:3.0
+ci_hosts="\
+192.168.0.212 \
+192.168.0.215 \
+192.168.0.217 \
+192.168.0.219 \
+"
+if [ -z "$info" ]; then
+    echo "PR-${pr_num}:${n} not found"
+    for host in $ci_hosts; do
+        ssh root@$host "sh -c \"grep -n \\\"^[0-9]\\\\\{8\\\\\}-[0-9]\\\\\{6\\\\\}\\\" /var/lib/jenkins/workspace/jenkins.log | grep \\\"PR-${pr_num}:${n}:\\\"\""
+        if [ $? -eq 0 ]; then
+            echo "this PR is possibly on host $host"
+            break
+        fi
+    done
+    exit 1
+fi
+line_num=`echo "$info"|wc -l`
+curr=`echo "$info"|head -n1`
+if [ "$line_num" == "2" ]; then
+    next=`echo "$info"|tail -n1`
+fi
+
+# check if it is TDinternal CI
+internal=0
+commit_prefix=community
+echo "$curr"|grep -q TDinternalCI
+if [ $? -eq 0 ]; then
+    internal=1
+    commit_prefix=tdinternal
+fi
+
+curr_line=`echo "$curr"|cut -d: -f1`
+next_line='$'
+if [ ! -z "$next" ]; then
+    next_line=`echo "$next"|cut -d: -f1`
+    next_line=$(( next_line - 1 ))
+fi
+# echo "$curr_line, $next_line"
+
+details=`sed -n "${curr_line},${next_line}p" $JENKINS_LOG`
+merge_line=`echo "$details"|grep -A 10 "$commit_prefix log merged: "|grep "Merge .* into"|head -n1`
+if [ -z "$merge_line" ]; then
+    echo "merge commit not found"
+    exit 1
+fi
+echo "$merge_line"
+branch=`echo "$merge_line"|awk '{print $2}'`
+commit_id=`echo "$merge_line"|awk '{print $4}'`
+# echo "$details"
+community_id=`echo "$details"|grep "community log: commit"|awk '{print $NF}'`
+internal_id=`echo "$details"|grep "tdinternal log: commit"|awk '{print $NF}'`
+python_connector_id=`echo "$details"|grep "python connector log: commit"|awk '{print $NF}'`
+# change_branch=`echo "$details"|grep "CHANGE_BRANCH"|sed "s/CHANGE_BRANCH://"`
+
+# if [ -z "${branch}" ]; then
+#     branch="$change_branch"
+# fi
+
+PWD=`pwd`
+log_dir=`ls log|grep "PR-${pr_num}_${n}_"`
+if [ -z "$log_dir" ]; then
+    echo "no log dir found"
+else
+    mount_dir="-v ${PWD}/log/$log_dir:/home/log"
+    build_dir=`ls log/$log_dir | grep "build_"`
+    if [ ! -z "$build_dir" ]; then
+        mount_dir="$mount_dir -v ${PWD}/log/$log_dir/$build_dir:/home/TDinternal/debug/build"
+    fi
+fi
+
+docker run -d --privileged -it --name $container_name \
+    $mount_dir \
+    taos_test:v1.0 bash
+
+if [ $internal -eq 0 ]; then
+    docker exec $container_name /home/setup.sh -c $commit_id -m $branch -n
+    echo "TDinternal checkout: $internal_id"
+    docker exec $container_name sh -c "cd /home/TDinternal; git checkout $internal_id"
+else
+    docker exec $container_name /home/setup.sh -e -c $commit_id -m $branch -n
+    echo "community checkout: $community_id"
+    docker exec $container_name sh -c "cd /home/TDinternal/community; git checkout $community_id"
+fi
+echo
+echo "* run the following command to enter the container:"
+echo "  docker exec -it $container_name bash"
+if [ -z "$log_dir" ]; then
+    echo "* no log dir found"
+else
+    echo "* log and coredump files are located in /home/log"
+fi
+if [ -z "$build_dir" ]; then
+    echo "* no build dir found"
+else
+    echo "* build files are located in /home/TDinternal/debug/build"
+fi
+echo "* source files are located in /home/TDinternal"
+
--- a/tests/ci/start_http.sh
+++ b/tests/ci/start_http.sh
@ -0,0 +1,16 @@
+#!/bin/bash
+
+script_dir=$(dirname $0)
+cd $script_dir || exit
+script_name=$(basename $0)
+ps -ef|grep -v grep|grep -v $$|grep -q "$script_name"
+if [ $? -eq 0 ]; then
+    exit 0
+fi
+while [ 1 ]; do
+    ps -ef|grep python|grep -q 8081
+    if [ $? -ne 0 ]; then
+        python3 -m http.server 8081
+    fi
+    sleep 60
+done
--- a/tests/parallel_test/container_build.sh
+++ b/tests/parallel_test/container_build.sh
@ -61,14 +61,28 @@ docker run \
    -v /root/go/pkg/mod:/root/go/pkg/mod \
    -v /root/.cache/go-build:/root/.cache/go-build \
    -v /root/.cos-local.1:/root/.cos-local.2 \
-    -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
-    -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
+    -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \
+    -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \
+    -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \
+    -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \
+    -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \
+    -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \
    -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
-    -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
    -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
+    -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \
+    -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \
+    -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \
+    -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
+    -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \
    -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
    -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+    -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \
+    -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \
+    -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \
+    -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \
+    -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
+    -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1"
 # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \

@ -96,15 +110,29 @@ docker run \
    -v /root/go/pkg/mod:/root/go/pkg/mod \
    -v /root/.cache/go-build:/root/.cache/go-build \
    -v /root/.cos-local.1:/root/.cos-local.2 \
-    -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \
+    -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \
+    -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \
+    -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \
+    -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \
    -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \
+    -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \
+    -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \
    -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
-    -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
    -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
+    -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \
+    -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \
+    -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \
+    -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
+    -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \
    -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
    -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+    -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \
+    -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \
+    -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \
+    -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \
+    -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-    -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
+    -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
    --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true   -DBUILD_SANITIZER=1  -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "

 mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugSan