feat: reusable workflow for taosd ci

Signed-off-by: WANG Xu <feici02@outlook.com>

.github/workflows/taosd-ci.yml

@@ -9,15 +9,12 @@ on:
     paths-ignore:
       - 'packaging/**'
       - 'docs/**'
-  repository_dispatch:
-    types: [run-tests]
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.event.client_payload.ref}}-${{ github.event_name == 'repository_dispatch' && 'dispatch' || ''}}
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.event.client_payload.ref }}-${{ github.event_name == 'repository_dispatch' && 'dispatch' || ''}}
   cancel-in-progress: true
 
 env:
-  CONTAINER_NAME: 'taosd-test'
   WKDIR: '/var/lib/jenkins/workspace'
   WK: '/var/lib/jenkins/workspace/TDinternal'
   WKC: '/var/lib/jenkins/workspace/TDinternal/community'
@@ -80,379 +77,34 @@ jobs:
           echo "pr_number=$pr_number" >> $GITHUB_OUTPUT
 
   run-tests-on-linux:
+    uses: taosdata/.github/.github/workflows/run-tests-on-linux.yml@feat/wangxu/taosd-ci
     needs: fetch-parameters
-    runs-on: 
-      group: CI
-      labels: [self-hosted, Linux, X64, testing]
-    timeout-minutes: 200
-    env:
-      IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
-      RUN_RUNCTION_TEST: ${{ needs.fetch-parameters.outputs.run_function_test }}
-      RUN_TDGPT_TEST: ${{ needs.fetch-parameters.outputs.run_tdgpt_test }}
-      SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
-      TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
-      PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
-    steps:
-      - name: Output the environment information
-        run: |
-          echo "::group::Environment Info"
-          date
-          hostname
-          env
-          echo "Runner: ${{ runner.name }}"
-          echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}"
-          echo "Workspace: ${{ env.WKDIR }}"
-          git --version
-          echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}"
-          echo "::endgroup::"
-
-      - name: Prepare repositories
-        run: |
-          set -euo pipefail
-          prepare_environment() {
-            cd "$1"
-            git reset --hard
-            git clean -f
-            git remote prune origin
-            git fetch
-            git checkout "$2"
-          }
-          prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}"
-          prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}"
-
-      - name: Get latest codes and logs for TDinternal PR
-        if: ${{ env.IS_TDINTERNAL == 'true' }}
-        run: |
-          cd ${{ env.WK }}
-          git pull >/dev/null
-          git log -5
-          echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
-          git checkout -qf FETCH_HEAD
-          git log -5
-          echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          cd ${{ env.WKC }}
-          git remote prune origin
-          git pull >/dev/null
-          git log -5
-          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-      - name: Get latest codes and logs for TDengine PR
-        if: ${{ env.IS_TDINTERNAL == 'false' }}
-        run: |
-          cd ${{ env.WKC }}
-          git remote prune origin
-          git pull >/dev/null
-          git log -5
-          echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
-          git checkout -qf FETCH_HEAD
-          git log -5
-          echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          cd ${{ env.WK }}
-          git pull >/dev/null
-          git log -5
-          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-      - name: Update submodule
-        run: |
-          cd ${{ env.WKC }}
-          git submodule update --init --recursive
-      - name: Output the 'file_no_doc_changed' information to the file
-        if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
-        run: |
-          mkdir -p ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}
-          cd ${{ env.WKC }}
-          changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ env.TARGET_BRANCH }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :)
-          echo $changed_files_non_doc > ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt
-      - name: Check assert testing
-        if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
-        run: |
-          cd ${{ env.WKC }}/tests/parallel_test
-          ./run_check_assert_container.sh -d ${{ env.WKDIR }}
-      - name: Check void function testing
-        if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
-        run: |
-          cd ${{ env.WKC }}/tests/parallel_test
-          ./run_check_void_container.sh -d ${{ env.WKDIR }}
-      - name: Build docker container
-        if: ${{ env.RUN_RUNCTION_TEST == 'true' }}
-        run: |
-          date
-          rm -rf ${{ env.WKC }}/debug
-          cd ${{ env.WKC }}/tests/parallel_test
-          time ./container_build.sh -w ${{ env.WKDIR }} -e
-      - name: Get parameters for testing
-        id: get_param
-        run: |
-          log_server_file="/home/log_server.json"
-          timeout_cmd=""
-          extra_param=""
-
-          if [ -f "$log_server_file" ]; then
-            log_server_enabled=$(jq '.enabled' "$log_server_file")
-            timeout_param=$(jq '.timeout' "$log_server_file")
-            if [ "$timeout_param" != "null" ] && [ "$timeout_param" != "0" ]; then
-              timeout_cmd="timeout $timeout_param"
-            fi
-
-            if [ "$log_server_enabled" == "1" ]; then
-              log_server=$(jq '.server' "$log_server_file" | sed 's/\\\"//g')
-              if [ "$log_server" != "null" ] && [ "$log_server" != "" ]; then
-                extra_param="-w $log_server"
-              fi
-            fi
-          fi
-          echo "timeout_cmd=$timeout_cmd" >> $GITHUB_OUTPUT
-          echo "extra_param=$extra_param" >> $GITHUB_OUTPUT
-      - name: Run function returns with a null pointer scan testing
-        if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' }}
-        run: |
-          cd ${{ env.WKC }}/tests/parallel_test
-          ./run_scan_container.sh -d ${{ env.WKDIR }} -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -f ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt ${{ steps.get_param.outputs.extra_param }}
-      - name: Run tdgpt test cases
-        if: ${{ env.IS_TDINTERNAL == 'false' && env.TARGET_BRANCH != '3.1' && env.RUN_TDGPT_TEST == 'true' }}
-        run: |
-          cd ${{ env.WKC }}/tests/parallel_test
-          export DEFAULT_RETRY_TIME=2
-          date
-          timeout 600 time ./run.sh -e -m /home/m.json -t tdgpt_cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 300 ${{ steps.get_param.outputs.extra_param }}
-      - name: Run function test cases
-        if: ${{ env.RUN_RUNCTION_TEST == 'true'}}
-        run: |
-          cd ${{ env.WKC }}/tests/parallel_test
-          export DEFAULT_RETRY_TIME=2
-          date
-          ${{ steps.get_param.outputs.timeout_cmd }} time ./run.sh -e -m /home/m.json -t cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 1200 ${{ steps.get_param.outputs.extra_param }}
+    if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }}
+    with:
+      tdinternal: ${{ needs.fetch-parameters.outputs.tdinternal }}
+      run_function_test: ${{ needs.fetch-parameters.outputs.run_function_test }}
+      run_tdgpt_test: ${{ needs.fetch-parameters.outputs.run_tdgpt_test }}
+      source_branch: ${{ needs.fetch-parameters.outputs.source_branch }}
+      target_branch: ${{ needs.fetch-parameters.outputs.target_branch }}
+      pr_number: ${{ needs.fetch-parameters.outputs.pr_number }}
 
   run-tests-on-mac:
+    uses: taosdata/.github/.github/workflows/run-tests-on-macos.yml@feat/wangxu/taosd-ci
     needs: fetch-parameters
     if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }}
-    runs-on: 
-      group: CI
-      labels: [self-hosted, macOS, testing]
-    timeout-minutes: 60
-    env:
-      IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
-      SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
-      TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
-      PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
-    steps:
-      - name: Output the environment information
-        run: |
-          echo "::group::Environment Info"
-          date
-          hostname
-          env
-          echo "Runner: ${{ runner.name }}"
-          echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}"
-          echo "Workspace: ${{ env.WKDIR }}"
-          git --version
-          echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}"
-          echo "::endgroup::"
-      - name: Prepare repositories
-        run: |
-          set -euo pipefail
-          prepare_environment() {
-            cd "$1"
-            git reset --hard
-            git clean -f
-            git remote prune origin
-            git fetch
-            git checkout "$2"
-          }
-          prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}"
-          prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}"
-      - name: Get latest codes and logs for TDinternal PR
-        if: ${{ env.IS_TDINTERNAL == 'true' }}
-        run: |
-          cd ${{ env.WK }}
-          git pull >/dev/null
-          git log -5
-          echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
-          git checkout -qf FETCH_HEAD
-          git log -5
-          echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          cd ${{ env.WKC }}
-          git remote prune origin
-          git pull >/dev/null
-          git log -5
-          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-      - name: Get latest codes and logs for TDengine PR
-        if: ${{ env.IS_TDINTERNAL == 'false' }}
-        run: |
-          cd ${{ env.WKC }}
-          git remote prune origin
-          git pull >/dev/null
-          git log -5
-          echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
-          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
-          git checkout -qf FETCH_HEAD
-          git log -5
-          echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-          cd ${{ env.WK }}
-          git pull >/dev/null
-          git log -5
-          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
-      - name: Update submodule
-        run: |
-          cd ${{ env.WKC }}
-          git submodule update --init --recursive
-      - name: Run tests
-        run: |
-          date
-          cd ${{ env.WK }}
-          rm -rf debug
-          mkdir debug
-          cd ${{ env.WK }}/debug
-          echo $PATH
-          echo "PATH=/opt/homebrew/bin:$PATH" >> $GITHUB_ENV
-          cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false  -DCMAKE_BUILD_TYPE=Release
-          make -j10
-          ctest -j10 || exit 7
-          date
+    with:
+      tdinternal: ${{ needs.fetch-parameters.outputs.tdinternal }}
+      source_branch: ${{ needs.fetch-parameters.outputs.source_branch }}
+      target_branch: ${{ needs.fetch-parameters.outputs.target_branch }}
+      pr_number: ${{ needs.fetch-parameters.outputs.pr_number }}
 
   run-tests-on-windows:
+    uses: taosdata/.github/.github/workflows/run-tests-on-windows.yml@feat/wangxu/taosd-ci
     needs: fetch-parameters
     if: ${{ needs.fetch-parameters.outputs.run_function_test == 'true' }}
-    runs-on:
-      group: CI
-      labels: [self-hosted, Windows, X64, testing]
-    timeout-minutes: 126
-    env:
-      IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
-      SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
-      TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
-      PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
-      WIN_INTERNAL_ROOT: "C:\\workspace\\0\\TDinternal"
-      WIN_COMMUNITY_ROOT: "C:\\workspace\\0\\TDinternal\\community"
-      WIN_SYSTEM_TEST_ROOT: "C:\\workspace\\0\\TDinternal\\community\\tests\\system-test"
-      WIN_VS_PATH: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat"
-      WIN_CPU_TYPE: "x64"
-    steps:
-      - name: Output the environment information
-        run: |
-          hostname
-          taskkill /f /t /im python.exe
-          taskkill /f /t /im bash.exe
-          taskkill /f /t /im taosd.exe
-          ipconfig
-          set
-          date /t
-          time /t
-          rd /s /Q "%WIN_INTERNAL_ROOT%\debug" || exit 0
-        shell: cmd
-      - name: Prepare repositories
-        run: |
-          :: Prepare internal repository
-          if exist "%WIN_INTERNAL_ROOT%" (
-            cd /d "%WIN_INTERNAL_ROOT%"
-            git reset --hard
-            git clean -f
-            git remote prune origin
-            git fetch
-            git checkout "%TARGET_BRANCH%"
-          ) else (
-            echo Directory does not exist: "%WIN_INTERNAL_ROOT%"
-            exit 1
-          )
+    with:
+      tdinternal: ${{ needs.fetch-parameters.outputs.tdinternal }}
+      source_branch: ${{ needs.fetch-parameters.outputs.source_branch }}
+      target_branch: ${{ needs.fetch-parameters.outputs.target_branch }}
+      pr_number: ${{ needs.fetch-parameters.outputs.pr_number }}
 
-          :: Prepare community repository
-          if exist "%WIN_COMMUNITY_ROOT%" (
-            cd /d "%WIN_COMMUNITY_ROOT%"
-            git reset --hard
-            git clean -f
-            git remote prune origin
-            git fetch
-            git checkout "%TARGET_BRANCH%"
-          ) else (
-            echo Directory does not exist: "%WIN_COMMUNITY_ROOT%"
-            exit 1
-          )
-        shell: cmd
-      - name: Get latest codes and logs for TDinternal PR
-        if: ${{ env.IS_TDINTERNAL == 'true' }}
-        run: |
-          cd %WIN_INTERNAL_ROOT%
-          git pull origin %TARGET_BRANCH%
-          git fetch origin +refs/pull/%PR_NUMBER%/merge
-          git checkout -qf FETCH_HEAD
-          cd %WIN_COMMUNITY_ROOT%
-          git remote prune origin
-          git pull
-        shell: cmd
-      - name: Get latest codes and logs for TDengine PR
-        if: ${{ env.IS_TDINTERNAL == 'false' }}
-        run: |
-          cd %WIN_INTERNAL_ROOT%
-          git pull origin %TARGET_BRANCH%
-          cd %WIN_COMMUNITY_ROOT%
-          git remote prune origin
-          git pull  origin %TARGET_BRANCH%
-          git fetch origin +refs/pull/%PR_NUMBER%/merge
-          git checkout -qf FETCH_HEAD
-        shell: cmd
-      - name: Output branch and log information
-        run: |
-          cd %WIN_INTERNAL_ROOT%
-          git branch
-          git log -5
-
-          cd %WIN_COMMUNITY_ROOT%
-          git branch
-          git log -5
-        shell: cmd
-      - name: Update submodule
-        run: |
-          cd %WIN_COMMUNITY_ROOT%
-          git submodule update --init --recursive
-        shell: cmd
-      - name: Build on windows
-        run: |
-          echo "building ..."
-          time /t
-          cd %WIN_INTERNAL_ROOT%
-          mkdir debug
-          cd debug
-          time /t
-          call "%WIN_VS_PATH%" %WIN_CPU_TYPE%
-          set CL=/MP8
-          echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake"
-          time /t
-          cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7
-          echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6"
-          time /t
-          jom -j 6 || exit 8
-          time /t
-
-          cd %WIN_COMMUNITY_ROOT%/tests/ci
-          pip3 install taospy==2.7.21 
-          pip3 install taos-ws-py==0.3.8
-          xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32
-        shell: cmd
-      - name: Run ctest
-        run: |
-          echo "windows ctest ..."
-          time /t
-          cd %WIN_INTERNAL_ROOT%\\debug
-          ctest -j 1 || exit 7
-          time /t
-        shell: cmd
-      - name: Run function test
-        run: |
-          echo "windows test ..."
-          xcopy /e/y/i/f "%WIN_INTERNAL_ROOT%\debug\build\lib\taos.dll" C:\Windows\System32
-          ls -l "C:\Windows\System32\taos.dll"
-          time /t
-          cd %WIN_SYSTEM_TEST_ROOT%
-          echo "testing ..."
-          test-all.bat ci
-          time /t
-        shell: cmd