Merge pull request #30435 from taosdata/test/cover1

test:add coverage workflow

.github/workflows/tdengine-test-coverage.yml

@@ -0,0 +1,28 @@
+# Coverage test for taosd & taosc
+	
+name: TDengine Coverage
+
+on:
+  workflow_dispatch:
+    inputs:
+      specified_test_branch:
+        description: 'Enter the test branch name of TDengine'
+        required: true
+        type: string
+  schedule:
+    - cron: '30 00 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.inputs.specified_test_branch || '3.0' }}-TDengine
+  cancel-in-progress: true
+
+env:
+  WKC: '/var/lib/jenkins/workspace/TDinternal/community'
+
+jobs:
+  run-tests-on-linux:
+    uses: taosdata/.github/.github/workflows/run-coverage-tests-on-linux.yml@main
+    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'}}
+    with:
+      tdinternal: false
+      specified_test_branch: ${{ github.event_name == 'schedule' && 'unavailable' || github.event.inputs.specified_test_branch  || '3.0' }}

tests/parallel_test/container_build_coverage.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+set -e 
+
+function usage() {
+    echo "$0"
+    echo -e "\t -w work dir"
+    echo -e "\t -e enterprise edition"
+    echo -e "\t -t make thread count"
+    echo -e "\t -h help"
+}
+
+ent=0
+while getopts "w:t:eh" opt; do
+    case $opt in
+        w)
+            WORKDIR=$OPTARG
+            ;;
+        e)
+            ent=1
+            ;;
+        t)
+            THREAD_COUNT=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$WORKDIR" ]; then
+    usage
+    exit 1
+fi
+
+ulimit -c unlimited
+
+if [ $ent -eq 0 ]; then
+    REP_DIR=/home/TDengine
+    REP_REAL_PATH=$WORKDIR/TDengine
+    REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDengine
+else
+    REP_DIR=/home/TDinternal
+    REP_REAL_PATH=$WORKDIR/TDinternal
+    REP_MOUNT_PARAM=$REP_REAL_PATH:/home/TDinternal
+    
+fi
+date 
+docker run \
+    -v $REP_MOUNT_PARAM \
+    -v /root/.cargo/registry:/root/.cargo/registry \
+    -v /root/.cargo/git:/root/.cargo/git \
+    -v /root/go/pkg/mod:/root/go/pkg/mod \
+    -v /root/.cache/go-build:/root/.cache/go-build \
+    -v /root/.cos-local.1:/root/.cos-local.2 \
+    -v ${REP_REAL_PATH}/enterprise/contrib/grant-lib:${REP_DIR}/enterprise/contrib/grant-lib \
+    -v ${REP_REAL_PATH}/community/tools/taosadapter:${REP_DIR}/community/tools/taosadapter \
+    -v ${REP_REAL_PATH}/community/tools/taosws-rs:${REP_DIR}/community/tools/taosws-rs \
+    -v ${REP_REAL_PATH}/community/contrib/apr/:${REP_DIR}/community/contrib/apr \
+    -v ${REP_REAL_PATH}/community/contrib/apr-util/:${REP_DIR}/community/contrib/apr-util \
+    -v ${REP_REAL_PATH}/community/contrib/cJson/:${REP_DIR}/community/contrib/cJson \
+    -v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
+    -v ${REP_REAL_PATH}/community/contrib/curl/:${REP_DIR}/community/contrib/curl \
+    -v ${REP_REAL_PATH}/community/contrib/curl2/:${REP_DIR}/community/contrib/curl2 \
+    -v ${REP_REAL_PATH}/community/contrib/geos/:${REP_DIR}/community/contrib/geos \
+    -v ${REP_REAL_PATH}/community/contrib/googletest/:${REP_DIR}/community/contrib/googletest \
+    -v ${REP_REAL_PATH}/community/contrib/libs3/:${REP_DIR}/community/contrib/libs3 \
+    -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
+    -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
+    -v ${REP_REAL_PATH}/community/contrib/lzma2/:${REP_DIR}/community/contrib/lzma2 \
+    -v ${REP_REAL_PATH}/community/contrib/mxml/:${REP_DIR}/community/contrib/mxml \
+    -v ${REP_REAL_PATH}/community/contrib/openssl/:${REP_DIR}/community/contrib/openssl \
+    -v ${REP_REAL_PATH}/community/contrib/pcre2/:${REP_DIR}/community/contrib/pcre2 \
+    -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
+    -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR; rm -rf debug; mkdir -p debug; cd debug; cmake .. -DCOVER=true -DBUILD_TEST=true -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_GEOS=true ; make -j install|| exit 1"
+
+
+if [[ -d ${WORKDIR}/debugNoSan  ]] ;then
+    echo "delete  ${WORKDIR}/debugNoSan"
+    rm -rf  ${WORKDIR}/debugNoSan
+fi
+if [[ -d ${WORKDIR}/debugRelease ]] ;then
+    echo "delete  ${WORKDIR}/debugRelease"
+    rm -rf  ${WORKDIR}/debugRelease
+fi
+
+mv  ${REP_REAL_PATH}/debug  ${WORKDIR}/debugNoSan
+
+
+ret=$?
+exit $ret
+

tests/parallel_test/run_container.sh

@@ -85,7 +85,7 @@ if [ $ent -ne 0 ]; then
     SIM_DIR=/home/TDinternal/sim
     REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal"
     REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/"
-    REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib:ro"
+    REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib"
 else
     # community edition
     REPDIR=$WORKDIR/TDengine
@@ -94,7 +94,7 @@ else
     SIM_DIR=/home/TDengine/sim
     REP_MOUNT_PARAM="$REPDIR:/home/TDengine"
     REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDengine/debug/"
-    REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib:ro"
+    REP_MOUNT_LIB="${REPDIR_DEBUG}/build/lib:/home/TDinternal/debug/build/lib"
 fi
 
 ulimit -c unlimited

tests/parallel_test/run_coverage_container.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+function usage() {
+    echo "$0"
+    echo -e "\t -d work dir"
+    echo -e "\t -b branch id"
+    echo -e "\t -h help"
+}
+
+while getopts "d:b:w:f:h" opt; do
+    case $opt in
+        d)
+            WORKDIR=$OPTARG
+            ;;
+        b)
+            branch_name_id=$OPTARG
+            ;;
+        h)
+            usage
+            exit 0
+            ;;
+        \?)
+            echo "Invalid option: -$OPTARG"
+            usage
+            exit 0
+            ;;
+    esac
+done
+
+if [ -z "$branch_name_id" ]; then
+    usage
+    exit 1
+fi
+
+if [ -z "$WORKDIR" ]; then
+    usage
+    exit 1
+fi
+
+ # enterprise edition
+INTERNAL_REPDIR=$WORKDIR/TDinternal
+REPDIR_DEBUG=$WORKDIR/debugNoSan/
+
+REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/"
+REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal"
+
+CONTAINER_TESTDIR=/home/TDinternal/community
+
+
+ulimit -c unlimited
+
+docker run \
+    --name taos_coverage \
+    -v $REP_MOUNT_PARAM \
+    -v $REP_MOUNT_DEBUG \
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "bash ${CONTAINER_TESTDIR}/tests/run_workflow_coverage.sh -b ${branch_name_id}" 
+
+
+ret=$?
+exit $ret
+

tests/run_workflow_coverage.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
+function print_color() {
+    local color="$1"
+    local message="$2"
+    echo -e "${color}${message}${NC}"
+}
+
+function printHelp() {
+    echo "Usage: $(basename $0) [options]"
+    echo
+    echo "Options:"
+    echo "    -d [Project dir]            Project directory (default: outermost project directory)"
+    echo "                                    e.g., -d /home/TDinternal/community"
+    echo "    -f [Capture gcda dir]       Capture gcda directory (default: <project dir>/debug)"
+    echo "    -b [Coverage branch]        Covevrage branch (default:3.0)"
+    exit 0
+}
+
+# Find the project/tdengine/build/capture directory 
+function get_DIR() {
+    today=`date +"%Y%m%d"`
+    if [ -z "$PROJECT_DIR" ]; then
+        CODE_DIR=$(dirname $0)
+        cd $CODE_DIR
+        CODE_DIR=$(pwd)
+        if [[ "$CODE_DIR" == *"/community/"*  ]]; then
+            PROJECT_DIR=$(realpath ../..)
+            TDENGINE_DIR="$PROJECT_DIR"
+            BUILD_DIR="$PROJECT_DIR/debug"
+            CAPTURE_GCDA_DIR="$BUILD_DIR"
+        else
+            PROJECT_DIR=$(realpath ..)
+            TDENGINE_DIR="$PROJECT_DIR"
+            BUILD_DIR="$PROJECT_DIR/debug"
+            CAPTURE_GCDA_DIR="$BUILD_DIR"
+        fi
+    elif [[ "$PROJECT_DIR" == *"/TDinternal" ]]; then
+        TDENGINE_DIR="$PROJECT_DIR/community"
+        BUILD_DIR="$PROJECT_DIR/debug"
+        CAPTURE_GCDA_DIR="$BUILD_DIR"
+    elif [[ "$PROJECT_DIR" == *"/TDengine" ]]; then
+        TDENGINE_DIR="$PROJECT_DIR"
+        BUILD_DIR="$PROJECT_DIR/debug"
+        CAPTURE_GCDA_DIR="$BUILD_DIR"
+    fi
+}
+
+function lcovFunc {
+    echo "collect data by lcov"
+    cd $TDENGINE_DIR
+
+    # collect data
+    lcov -d "$CAPTURE_GCDA_DIR" -capture --rc lcov_branch_coverage=1 --rc genhtml_branch_coverage=1 --no-external -b $TDENGINE_DIR -o coverage.info
+
+    # remove exclude paths 
+    lcov --remove coverage.info \
+        '*/contrib/*' '*/test/*' '*/packaging/*' '*/taos-tools/deps/*' '*/taosadapter/*' '*/TSZ/*' \
+        '*/AccessBridgeCalls.c' '*/ttszip.c' '*/dataInserter.c' '*/tlinearhash.c' '*/tsimplehash.c' '*/tsdbDiskData.c' '/*/enterprise/*' '*/docs/*' '*/sim/*'\
+        '*/texpr.c' '*/runUdf.c' '*/schDbg.c' '*/syncIO.c' '*/tdbOs.c' '*/pushServer.c' '*/osLz4.c'\
+        '*/tbase64.c' '*/tbuffer.c' '*/tdes.c' '*/texception.c' '*/examples/*' '*/tidpool.c' '*/tmempool.c'\
+        '*/clientJniConnector.c' '*/clientTmqConnector.c' '*/version.cc' '*/strftime.c' '*/localtime.c'\
+        '*/tthread.c' '*/tversion.c'  '*/ctgDbg.c' '*/schDbg.c' '*/qwDbg.c' '*/version.c' '*/tencode.h' \
+        '*/shellAuto.c' '*/shellTire.c' '*/shellCommand.c' '*/debug/*' '*/tests/*'\
+        '*/tsdbFile.c' '*/tsdbUpgrade.c' '*/tsdbFS.c' '*/tsdbReaderWriter.c' \ 
+        '*/sql.c' '*/sql.y' '*/smaSnapshot.c' '*/smaCommit.c'\
+         --rc lcov_branch_coverage=1  -o coverage.info
+
+    # generate result
+    echo "generate result"
+    lcov -l --rc lcov_branch_coverage=1 coverage.info 
+
+    sed -i 's/\/home\/TDinternal\/community\/sql.c/\/home\/TDinternal/\community\/source\/libs\/parser\/inc\/sql.c/g' coverage.info
+    sed -i 's/\/home\/TDinternal\/community\/sql.y/\/home\/TDinternal/\community\/source\/libs\/parser\/inc\/sql.y/g' coverage.info
+
+    # push result to coveralls.io
+    echo "push result to coveralls.io"
+    /usr/local/bin/coveralls-lcov -t WOjivt0JCvDfqHDpyBQXtqhYbOGANrrps -b $BRANCH $TDENGINE_DIR/coverage.info
+}
+
+
+######################
+# main entry
+######################
+
+# Initialization parameter
+PROJECT_DIR=""
+CAPTURE_GCDA_DIR=""
+BRANCH=""
+
+# Parse command line parameters
+while getopts "hd:b:f:" arg; do
+  case $arg in
+    d)
+      PROJECT_DIR=$OPTARG
+      ;;
+    b)
+      BRANCH=$OPTARG
+      ;;
+    f)
+      CAPTURE_GCDA_DIR=$OPTARG
+      ;;
+    h)
+      printHelp
+      ;;
+    ?)
+      echo "Usage: ./$(basename $0) -h"
+      exit 1
+      ;;
+  esac
+done
+
+# Show all parameters
+print_color "$GREEN" "Run coverage test on workflow!"
+get_DIR
+echo "PROJECT_DIR = $PROJECT_DIR"
+echo "TDENGINE_DIR = $TDENGINE_DIR"
+echo "BUILD_DIR = $BUILD_DIR"
+echo "CAPTURE_GCDA_DIR = $CAPTURE_GCDA_DIR"
+
+lcovFunc
+print_color "$GREEN" "End of coverage test on workflow!"