diff --git a/tests/ci/ci_deploy_dependency_file_list.txt b/tests/ci/ci_deploy_dependency_file_list.txt new file mode 100644 index 0000000000..1e7afdd7db --- /dev/null +++ b/tests/ci/ci_deploy_dependency_file_list.txt @@ -0,0 +1,8 @@ +/home/m.json +/home/log_server.json +/var/lib/jenkins/workspace/restore.sh +/var/lib/jenkins/workspace/start_http.sh +/var/lib/jenkins/workspace/TDinternal +/var/lib/jenkins/workspace/remove_corefile.sh +/var/lib/jenkins/workspace/CI_disk_Monitor.py +/var/lib/jenkins/workspace/start_CI_Monitor.sh diff --git a/tests/ci/container_build_newmachine.sh b/tests/ci/container_build_newmachine.sh new file mode 100755 index 0000000000..3c87cf156f --- /dev/null +++ b/tests/ci/container_build_newmachine.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +function usage() { + echo "$0" + echo -e "\t -w work dir" + echo -e "\t -e enterprise edition" + echo -e "\t -t make thread count" + echo -e "\t -h help" +} + +ent=0 +while getopts "w:t:eh" opt; do + case $opt in + w) + WORKDIR=$OPTARG + ;; + e) + ent=1 + ;; + t) + THREAD_COUNT=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 0 + ;; + esac +done + +if [ -z "$WORKDIR" ]; then + usage + exit 1 +fi +# if [ -z "$THREAD_COUNT" ]; then +# THREAD_COUNT=1 +# fi + +ulimit -c unlimited + +if [ $ent -eq 0 ]; then + REP_DIR=/home/TDengine + REP_REAL_PATH=$WORKDIR/TDengine + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine +else + REP_DIR=/home/TDinternal + REP_REAL_PATH=$WORKDIR/TDinternal + REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal + +fi +date +docker run \ + -v $REP_MOUNT_PARAM \ + -v /root/.cargo/registry:/root/.cargo/registry \ + -v /root/.cargo/git:/root/.cargo/git \ + -v /root/go/pkg/mod:/root/go/pkg/mod \ + -v /root/.cache/go-build:/root/.cache/go-build \ + -v /root/.cos-local.1:/root/.cos-local.2 \ + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " + # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ + +if [[ -d ${WORKDIR}/debugNoSan ]] ;then + echo "delete ${WORKDIR}/debugNoSan" + rm -rf ${WORKDIR}/debugNoSan +fi +if [[ -d ${WORKDIR}/debugSan ]] ;then + echo "delete ${WORKDIR}/debugSan" + rm -rf ${WORKDIR}/debugSan +fi + +if [ "$(uname -m)" = "aarch64" ] ;then + CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Debug" +else + CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=Release" +fi + + +mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan +date + +ret=$? +exit $ret + diff --git a/tests/ci/deploy_ci_machines.sh b/tests/ci/deploy_ci_machines.sh new file mode 100644 index 0000000000..e34ec1d6e5 --- /dev/null +++ b/tests/ci/deploy_ci_machines.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +function scp_file_from_host { + # check at least three parameters + if [ "$#" -lt 3 ]; then + echo "Usage: $0 host passwd source_filename [dest_filename]" + exit 1 + fi + + host=$1 + passwd=$2 + source_filename=$3 + # If the fourth parameter is not provided, use the third parameter as the default value + dest_filename=${4:-$3} + + # use sshpass and scp for secure file transfer + sshpass -p "$passwd" scp -o StrictHostKeyChecking=no -r "$host":"$source_filename" "$dest_filename" +} + + +# install docker and sshpass +curl -fsSL https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository "deb [arch=amd64] https://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu $(lsb_release -cs) stable" +sudo apt-get update +sudo apt-get install -y docker-ce sshpass jq +sudo systemctl enable docker +sudo systemctl start docker + +# create a log directory +mkdir -p /var/lib/jenkins/workspace/log + +# Assuming you have a file called 'file_list.txt' with one filename per line +file_list="ci_deploy_dependency_file_list.txt" +monitorip="192.168.1.59" +passwd_all="abcdefg" + +# Read the file list and call scp_file_from_host for each file +while IFS= read -r source_filename; do + scp_file_from_host "$monitorip" "$passwd_all" "$source_filename" +done < "$file_list" + +# modify the configuration file +ip=$(ifconfig |grep inet|grep 192 |awk '{print $2}') +sed -i "s/${monitorip}/$ip/" /home/log_server.json +sed -i "s/${monitorip}/$ip/" /home/m.json + +#mkdir corefile dir and configure the system to automatically set corefile dir at startup +mkdir -p /home/coredump/ && echo "echo '/home/coredump/core_%e-%p' | sudo tee /proc/sys/kernel/core_pattern " >> /root/.bashrc + + +# get image from 0.212 +image_ip="192.168.0.212" +scp_file_from_host $image_ip $passwd_all "/home/tang/work/image/taos_image.tar " "/home/taos_image.tar" +docker load -i /home/taos_image.tar + +#start http server +nohup /var/lib/jenkins/workspace/log/start_http.sh & + +# start CI monitor and remove corefile in crontable +(crontab -l;echo "0 1 * * * /usr/bin/bash /var/lib/jenkins/workspace/remove_corefile.sh") | crontab +(crontab -l;echo "@reboot /usr/bin/bash /var/lib/jenkins/workspace/start_CI_Monitor.sh") | crontab + + +# generate cache dir +cd /var/lib/jenkins/workspace/TDinternal/community/tests/parallel_test || exit +time ./container_build_newmachine.sh -w /var/lib/jenkins/workspace -e + +# test if the CI machine compilation is successful +time ./container_build.sh -w /var/lib/jenkins/workspace -e \ No newline at end of file diff --git a/tests/ci/remove_corefile.sh b/tests/ci/remove_corefile.sh new file mode 100644 index 0000000000..7374101547 --- /dev/null +++ b/tests/ci/remove_corefile.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +date_str=$(date -d "4 day ago" +%Y%m%d) +if [ ! -z "$1" ]; then + date_str="$1" +fi +script_dir=$(dirname $0) +cd "${script_dir}"/log || exit +# date >>core.list +# find . -name "core.*" | grep "$date_str" >>core.list +# find . -name "core.*" | grep "$date_str" | xargs rm -rf +# find . -name "build_*" | grep "$date_str" | xargs rm -rf +for file in *; do + if [[ $file == *"$date_str"* ]]; then + rm -rf "$file" + fi +done diff --git a/tests/ci/restore.sh b/tests/ci/restore.sh new file mode 100644 index 0000000000..4c9daf73fe --- /dev/null +++ b/tests/ci/restore.sh @@ -0,0 +1,159 @@ +#!/bin/bash + set -x +JENKINS_LOG=jenkins.log +# pr_num=14228 +# n=1 +function usage() { + echo "$0" + echo -e "\t -p PR number" + echo -e "\t -n build number" + echo -e "\t -c container name" + echo -e "\t -h help" +} +while getopts "p:n:c:h" opt; do + case $opt in + p) + pr_num=$OPTARG + ;; + n) + n=$OPTARG + ;; + c) + container_name=$OPTARG + ;; + h) + usage + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" + usage + exit 1 + ;; + esac +done +if [ -z "$container_name" ]; then + echo "container name not specified" + usage + exit 1 +fi +if [ -z "$pr_num" ]; then + echo "PR number not specified" + usage + exit 1 +fi +if [ -z "$n" ]; then + echo "build number not specified" + usage + exit 1 +fi +pr_num=`echo "$pr_num"|sed "s/PR-//"` +container_count=`docker ps -a -f name=$container_name|wc -l` +if [ $container_count -gt 1 ]; then + docker ps -a -f name=$container_name + echo "container $container_name exists" + exit 1 +fi +cd $(dirname $0) +info=`grep -n "^[0-9]\{8\}-[0-9]\{6\}" jenkins.log | grep -A 1 "PR-${pr_num}:${n}:"` +# 22131:20220625-113105 NewTest/PR-14228:PR-14228:1:3.0 +# 22270:20220625-121154 NewTest/PR-14221:PR-14221:2:3.0 +ci_hosts="\ +192.168.0.212 \ +192.168.0.215 \ +192.168.0.217 \ +192.168.0.219 \ +" +if [ -z "$info" ]; then + echo "PR-${pr_num}:${n} not found" + for host in $ci_hosts; do + ssh root@$host "sh -c \"grep -n \\\"^[0-9]\\\\\{8\\\\\}-[0-9]\\\\\{6\\\\\}\\\" /var/lib/jenkins/workspace/jenkins.log | grep \\\"PR-${pr_num}:${n}:\\\"\"" + if [ $? -eq 0 ]; then + echo "this PR is possibly on host $host" + break + fi + done + exit 1 +fi +line_num=`echo "$info"|wc -l` +curr=`echo "$info"|head -n1` +if [ "$line_num" == "2" ]; then + next=`echo "$info"|tail -n1` +fi + +# check if it is TDinternal CI +internal=0 +commit_prefix=community +echo "$curr"|grep -q TDinternalCI +if [ $? -eq 0 ]; then + internal=1 + commit_prefix=tdinternal +fi + +curr_line=`echo "$curr"|cut -d: -f1` +next_line='$' +if [ ! -z "$next" ]; then + next_line=`echo "$next"|cut -d: -f1` + next_line=$(( next_line - 1 )) +fi +# echo "$curr_line, $next_line" + +details=`sed -n "${curr_line},${next_line}p" $JENKINS_LOG` +merge_line=`echo "$details"|grep -A 10 "$commit_prefix log merged: "|grep "Merge .* into"|head -n1` +if [ -z "$merge_line" ]; then + echo "merge commit not found" + exit 1 +fi +echo "$merge_line" +branch=`echo "$merge_line"|awk '{print $2}'` +commit_id=`echo "$merge_line"|awk '{print $4}'` +# echo "$details" +community_id=`echo "$details"|grep "community log: commit"|awk '{print $NF}'` +internal_id=`echo "$details"|grep "tdinternal log: commit"|awk '{print $NF}'` +python_connector_id=`echo "$details"|grep "python connector log: commit"|awk '{print $NF}'` +# change_branch=`echo "$details"|grep "CHANGE_BRANCH"|sed "s/CHANGE_BRANCH://"` + +# if [ -z "${branch}" ]; then +# branch="$change_branch" +# fi + +PWD=`pwd` +log_dir=`ls log|grep "PR-${pr_num}_${n}_"` +if [ -z "$log_dir" ]; then + echo "no log dir found" +else + mount_dir="-v ${PWD}/log/$log_dir:/home/log" + build_dir=`ls log/$log_dir | grep "build_"` + if [ ! -z "$build_dir" ]; then + mount_dir="$mount_dir -v ${PWD}/log/$log_dir/$build_dir:/home/TDinternal/debug/build" + fi +fi + +docker run -d --privileged -it --name $container_name \ + $mount_dir \ + taos_test:v1.0 bash + +if [ $internal -eq 0 ]; then + docker exec $container_name /home/setup.sh -c $commit_id -m $branch -n + echo "TDinternal checkout: $internal_id" + docker exec $container_name sh -c "cd /home/TDinternal; git checkout $internal_id" +else + docker exec $container_name /home/setup.sh -e -c $commit_id -m $branch -n + echo "community checkout: $community_id" + docker exec $container_name sh -c "cd /home/TDinternal/community; git checkout $community_id" +fi +echo +echo "* run the following command to enter the container:" +echo " docker exec -it $container_name bash" +if [ -z "$log_dir" ]; then + echo "* no log dir found" +else + echo "* log and coredump files are located in /home/log" +fi +if [ -z "$build_dir" ]; then + echo "* no build dir found" +else + echo "* build files are located in /home/TDinternal/debug/build" +fi +echo "* source files are located in /home/TDinternal" + diff --git a/tests/ci/start_http.sh b/tests/ci/start_http.sh new file mode 100644 index 0000000000..f73ffc85c8 --- /dev/null +++ b/tests/ci/start_http.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +script_dir=$(dirname $0) +cd $script_dir || exit +script_name=$(basename $0) +ps -ef|grep -v grep|grep -v $$|grep -q "$script_name" +if [ $? -eq 0 ]; then + exit 0 +fi +while [ 1 ]; do + ps -ef|grep python|grep -q 8081 + if [ $? -ne 0 ]; then + python3 -m http.server 8081 + fi + sleep 60 +done \ No newline at end of file diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 3d3069f05f..85e3d2ab73 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -61,14 +61,28 @@ docker run \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ - -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ @@ -96,15 +110,29 @@ docker run \ -v /root/go/pkg/mod:/root/go/pkg/mod \ -v /root/.cache/go-build:/root/.cache/go-build \ -v /root/.cos-local.1:/root/.cos-local.2 \ - -v ${REP_REAL_PATH}/enterprise/src/plugins/taosx/target:${REP_DIR}/enterprise/src/plugins/taosx/target \ + -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \ + -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \ + -v ${REP_REAL_PATH}/community/tools/taos-tools:${REP_DIR}/community/tools/taos-tools \ + -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \ -v ${REP_REAL_PATH}/community/tools/taosws-rs/target:${REP_DIR}/community/tools/taosws-rs/target \ + -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \ + -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \ -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \ - -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \ + -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \ + -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \ + -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \ + -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \ + -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ + -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \ + -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \ + -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \ + -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \ + -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ + -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=false -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true $CMAKE_BUILD_TYPE -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan