Merge pull request #29967 from taosdata/workflow/ci

ci: add linux/mac ci build and test workflow

.github/workflows/taosd-ci.yml

@@ -0,0 +1,315 @@
+name: TDengine CI Test
+
+on:
+  pull_request:
+    branches:
+      - 'main'
+      - '3.0'
+      - '3.1'
+    paths-ignore:
+      - 'packaging/**'
+      - 'docs/**'
+  repository_dispatch:
+    types: [run-tests]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CONTAINER_NAME: 'taosd-test'
+  WKDIR: '/var/lib/jenkins/workspace'
+  WK: '/var/lib/jenkins/workspace/TDinternal'
+  WKC: '/var/lib/jenkins/workspace/TDinternal/community'
+
+jobs:
+  fetch-parameters:
+    runs-on: 
+      group: CI
+      labels: [self-hosted, Linux, X64, testing]
+    outputs:
+      tdinternal: ${{ steps.parameters.outputs.tdinternal }}
+      run_function_test: ${{ steps.parameters.outputs.run_function_test }}
+      run_tdgpt_test: ${{ steps.parameters.outputs.run_tdgpt_test }}
+      source_branch: ${{ steps.parameters.outputs.source_branch }}
+      target_branch: ${{ steps.parameters.outputs.target_branch }}
+      pr_number: ${{ steps.parameters.outputs.pr_number }}
+    steps:
+      - name: Determine trigger source and fetch parameters
+        id: parameters
+        run: |
+          set -euo pipefail
+          # check the trigger source and get branch information
+          if [ "${{ github.event_name }}" == "repository_dispatch" ]; then
+            tdinternal="true"
+            source_branch=${{ github.event.client_payload.tdinternal_source_branch }}
+            target_branch=${{ github.event.client_payload.tdinternal_target_branch }}
+            pr_number=${{ github.event.client_payload.tdinternal_pr_number }}
+            run_tdgpt_test="true"
+            run_function_test="true"
+          else
+            tdinternal="false"
+            source_branch=${{ github.event.pull_request.head.ref }}
+            target_branch=${{ github.event.pull_request.base.ref }}
+            pr_number=${{ github.event.pull_request.number }}
+
+            # check whether to run tdgpt test cases
+            cd ${{ env.WKC }}
+            changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD $target_branch`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :)
+
+            if [[ "$changed_files_non_doc" != '' &&  "$changed_files_non_doc" =~ /forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics/ ]]; then
+              run_tdgpt_test="true"
+            else
+              run_tdgpt_test="false"
+            fi
+
+            # check whether to run function test cases
+            changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD $target_branch`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -Ev "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" | tr '\n' ' ' ||:)
+            if [ $changed_files_non_tdgpt != '' ]; then
+              run_function_test="true"
+            else
+              run_function_test="false"
+            fi
+          fi
+
+          echo "tdinternal=$tdinternal" >> $GITHUB_OUTPUT
+          echo "run_function_test=$run_function_test" >> $GITHUB_OUTPUT
+          echo "run_tdgpt_test=$run_tdgpt_test" >> $GITHUB_OUTPUT
+          echo "source_branch=$source_branch" >> $GITHUB_OUTPUT
+          echo "target_branch=$target_branch" >> $GITHUB_OUTPUT
+          echo "pr_number=$pr_number" >> $GITHUB_OUTPUT
+
+  run-tests-on-linux:
+    needs: fetch-parameters
+    runs-on: 
+      group: CI
+      labels: [self-hosted, Linux, X64, testing]
+    timeout-minutes: 200
+    env:
+      IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
+      RUN_RUNCTION_TEST: ${{ needs.fetch-parameters.outputs.run_function_test }}
+      RUN_TDGPT_TEST: ${{ needs.fetch-parameters.outputs.run_tdgpt_tests }}
+      SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
+      TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
+      PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
+    steps:
+      - name: Output the environment information
+        run: |
+          echo "::group::Environment Info"
+          date
+          hostname
+          env
+          echo "Runner: ${{ runner.name }}"
+          echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}"
+          echo "Workspace: ${{ env.WKDIR }}"
+          git --version
+          echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}"
+          echo "::endgroup::"
+
+      - name: Prepare repositories
+        run: |
+          set -euo pipefail
+          prepare_environment() {
+            cd "$1"
+            git reset --hard
+            git clean -f
+            git remote prune origin
+            git fetch
+            git checkout "$2"
+          }
+          prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}"
+          prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}"
+
+      - name: Get latest codes and logs for TDinternal PR
+        if: ${{ env.IS_TDINTERNAL == 'true' }}
+        run: |
+          cd ${{ env.WK }}
+          git pull >/dev/null
+          git log -5
+          echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
+          git checkout -qf FETCH_HEAD
+          git log -5
+          echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          cd ${{ env.WKC }}
+          git remote prune origin
+          git pull >/dev/null
+          git log -5
+          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+      - name: Get latest codes and logs for TDengine PR
+        if: ${{ env.IS_TDINTERNAL == 'false' }}
+        run: |
+          cd ${{ env.WKC }}
+          git remote prune origin
+          git pull >/dev/null
+          git log -5
+          echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
+          git checkout -qf FETCH_HEAD
+          git log -5
+          echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          cd ${{ env.WK }}
+          git pull >/dev/null
+          git log -5
+          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+      - name: Update submodule
+        run: |
+          cd ${{ env.WKC }}
+          git submodule update --init --recursive
+      - name: Output the 'file_no_doc_changed' information to the file
+        if: ${{ env.IS_TDINTERNAL == 'false' }}
+        run: |
+          mkdir -p ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}
+          changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ env.TARGET_BRANCH }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :)
+          echo $changed_files_non_doc > ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt
+      - name: Check assert testing
+        run: |
+          cd ${{ env.WKC }}/tests/parallel_test
+          ./run_check_assert_container.sh -d ${{ env.WKDIR }}
+      - name: Check void function testing
+        run: |
+          cd ${{ env.WKC }}/tests/parallel_test
+          ./run_check_void_container.sh -d ${{ env.WKDIR }}
+      - name: Build docker container
+        run: |
+          date
+          rm -rf ${{ env.WKC }}/debug
+          cd ${{ env.WKC }}/tests/parallel_test
+          time ./container_build.sh -w ${{ env.WKDIR }} -e
+      - name: Get parameters for testing
+        id: get_param
+        run: |
+          log_server_file="/home/log_server.json"
+          timeout_cmd=""
+          extra_param=""
+
+          if [ -f "$log_server_file" ]; then
+            log_server_enabled=$(jq '.enabled' "$log_server_file")
+            timeout_param=$(jq '.timeout' "$log_server_file")
+            if [ "$timeout_param" != "null" ] && [ "$timeout_param" != "0" ]; then
+              timeout_cmd="timeout $timeout_param"
+            fi
+
+            if [ "$log_server_enabled" == "1" ]; then
+              log_server=$(jq '.server' "$log_server_file" | sed 's/\\\"//g')
+              if [ "$log_server" != "null" ] && [ "$log_server" != "" ]; then
+                extra_param="-w $log_server"
+              fi
+            fi
+          fi
+          echo "timeout_cmd=$timeout_cmd" >> $GITHUB_OUTPUT
+          echo "extra_param=$extra_param" >> $GITHUB_OUTPUT
+      - name: Run function returns with a null pointer scan testing
+        run: |
+          cd ${{ env.WKC }}/tests/parallel_test
+          ./run_scan_container.sh -d ${{ env.WKDIR }} -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -f ${{ env.WKDIR }}/tmp/${{ env.PR_NUMBER }}_${{ github.run_number }}/docs_changed.txt ${{ steps.get_param.outputs.extra_param }}
+      - name: Run tdgpt test cases
+        if: ${{ env.IS_TDINTERNAL }} == 'false' && ${{ env.RUN_TDGPT_TEST }} == 'true'
+        run: |
+          cd ${{ env.WKC }}/tests/parallel_test
+          export DEFAULT_RETRY_TIME=2
+          date
+          timeout 600 time ./run.sh -e -m /home/m.json -t tdgpt_cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 300 ${{ steps.get_param.outputs.extra_param }}
+      - name: Run function test cases
+        if: ${{ env.RUN_RUNCTION_TEST }} == 'true'
+        run: |
+          cd ${{ env.WKC }}/tests/parallel_test
+          export DEFAULT_RETRY_TIME=2
+          date
+          ${{ steps.get_param.outputs.timeout_cmd }} time ./run.sh -e -m /home/m.json -t cases.task -b ${{ env.PR_NUMBER }}_${{ github.run_number }} -l ${{ env.WKDIR }}/log -o 1200 ${{ steps.get_param.outputs.extra_param }}
+
+  run-tests-on-mac:
+    needs: fetch-parameters
+    if: ${{ needs.fetch-parameters.outputs.run_function_test == 'false' }}
+    runs-on: 
+      group: CI
+      labels: [self-hosted, macOS, ARM64, testing]
+    timeout-minutes: 60
+    env:
+      IS_TDINTERNAL: ${{ needs.fetch-parameters.outputs.tdinternal }}
+      SOURCE_BRANCH: ${{ needs.fetch-parameters.outputs.source_branch }}
+      TARGET_BRANCH: ${{ needs.fetch-parameters.outputs.target_branch }}
+      PR_NUMBER: ${{ needs.fetch-parameters.outputs.pr_number }}
+    steps:
+      - name: Output the environment information
+        run: |
+          echo "::group::Environment Info"
+          date
+          hostname
+          env
+          echo "Runner: ${{ runner.name }}"
+          echo "Trigger Source from TDinternal: ${{ env.IS_TDINTERNAL }}"
+          echo "Workspace: ${{ env.WKDIR }}"
+          git --version
+          echo "${{ env.WKDIR }}/restore.sh -p ${{ env.PR_NUMBER }} -n ${{ github.run_number }} -c ${{ env.CONTAINER_NAME }}"
+          echo "::endgroup::"
+      - name: Prepare repositories
+        run: |
+          set -euo pipefail
+          prepare_environment() {
+            cd "$1"
+            git reset --hard
+            git clean -f
+            git remote prune origin
+            git fetch
+            git checkout "$2"
+          }
+          prepare_environment "${{ env.WK }}" "${{ env.TARGET_BRANCH }}"
+          prepare_environment "${{ env.WKC }}" "${{ env.TARGET_BRANCH }}"
+      - name: Get latest codes and logs for TDinternal PR
+        if: ${{ env.IS_TDINTERNAL == 'true' }}
+        run: |
+          cd ${{ env.WK }}
+          git pull >/dev/null
+          git log -5
+          echo "`date "+%Y%m%d-%H%M%S"` TDinternalTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
+          git checkout -qf FETCH_HEAD
+          git log -5
+          echo "TDinternal log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          cd ${{ env.WKC }}
+          git remote prune origin
+          git pull >/dev/null
+          git log -5
+          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+      - name: Get latest codes and logs for TDengine PR
+        if: ${{ env.IS_TDINTERNAL == 'false' }}
+        run: |
+          cd ${{ env.WKC }}
+          git remote prune origin
+          git pull >/dev/null
+          git log -5
+          echo "`date "+%Y%m%d-%H%M%S"` TDengineTest/${{ env.PR_NUMBER }}:${{ github.run_number }}:${{ env.TARGET_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "CHANGE_BRANCH:${{ env.SOURCE_BRANCH }}" >>${{ env.WKDIR }}/jenkins.log
+          echo "community log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          git fetch origin +refs/pull/${{ env.PR_NUMBER }}/merge
+          git checkout -qf FETCH_HEAD
+          git log -5
+          echo "community log merged: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+          cd ${{ env.WK }}
+          git pull >/dev/null
+          git log -5
+          echo "TDinternal log: `git log -5`" >>${{ env.WKDIR }}/jenkins.log
+      - name: Update submodule
+        run: |
+          cd ${{ env.WKC }}
+          git submodule update --init --recursive
+      - name: Run tests
+        run: |
+          date
+          cd ${{ env.WK }}
+          rm -rf debug
+          mkdir debug
+          cd ${{ env.WK }}/debug
+          echo $PATH
+          echo "PATH=/opt/homebrew/bin:$PATH" >> $GITHUB_ENV
+          cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false  -DCMAKE_BUILD_TYPE=Release
+          make -j10
+          ctest -j10 || exit 7
+          date

.github/workflows/taosd-doc-build.yml

@@ -1,17 +1,14 @@
 name: TDengine Doc Build
 
 on:
-  workflow_call:
-    inputs:
-      target_branch:
-        description: "Target branch name of for building the document"
-        required: true
-        type: string
-
-      target_pr_number:
-        description: "PR number of target branch to merge for building the document"
-        required: true
-        type: string
+  pull_request:
+    branches:
+      - 'main'
+      - '3.0'
+      - '3.1'
+    paths:
+      - 'docs/**'
+      - '*.md'
 
 env:
   DOC_WKC: "/root/doc_ci_work"
@@ -21,81 +18,32 @@ env:
   TOOLS_REPO: "taos-tools"
 
 jobs:
-  check:
-    runs-on: 
-      group: CI
-      labels: [self-hosted, doc-build]
-    outputs:
-      changed_files_zh: ${{ steps.set_output.outputs.changed_files_zh }}
-      changed_files_en: ${{ steps.set_output.outputs.changed_files_en }}
-      changed_files_non_doc: ${{ steps.set_output.outputs.changed_files_non_doc }}
-      changed_files_non_tdgpt: ${{ steps.set_output.outputs.changed_files_non_tdgpt }}
-    steps:
-      - name: Get the latest document contents from the repository
-        run: |
-          set -e
-          # ./.github/scripts/update_repo.sh ${{ env.DOC_WKC }}/${{ env.TD_REPO }} ${{ inputs.target_branch }} ${{ inputs.target_pr_number }}
-          cd ${{ env.DOC_WKC }}/${{ env.TD_REPO }}
-          git reset --hard
-          git clean -f
-          git remote prune origin
-          git fetch
-          git checkout ${{ inputs.target_branch }}
-          git pull >/dev/null
-          git fetch origin +refs/pull/${{ inputs.target_pr_number }}/merge
-          git checkout -qf FETCH_HEAD
-      - name: Check whether the document is changed and set output variables
-        id: set_output
-        run: |
-          set -e
-          cd ${{ env.DOC_WKC }}/${{ env.TD_REPO }}
-          changed_files_zh=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ inputs.tartget_branch }}`| grep  "^docs/zh/" | tr '\n' ' ' || :)
-          changed_files_en=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ inputs.tartget_branch }}`| grep  "^docs/en/" | tr '\n' ' ' || :)
-          changed_files_non_doc=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ inputs.tartget_branch }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | tr '\n' ' ' || :)
-          changed_files_non_tdgpt=$(git --no-pager diff --name-only FETCH_HEAD `git merge-base FETCH_HEAD ${{ inputs.tartget_branch }}`|grep -v "^docs/en/"|grep -v "^docs/zh/"|grep -v ".md$" | grep -Ev "forecastoperator.c|anomalywindowoperator.c|tanalytics.h|tanalytics.c|tdgpt_cases.task|analytics" | tr '\n' ' ' ||:)
-          echo "changed_files_zh=${changed_files_zh}" >> $GITHUB_OUTPUT
-          echo "changed_files_en=${changed_files_en}" >> $GITHUB_OUTPUT
-          echo "changed_files_non_doc=${changed_files_non_doc}" >> $GITHUB_OUTPUT
-          echo "changed_files_non_tdgpt=${changed_files_non_tdgpt}" >> $GITHUB_OUTPUT
-
-  build:
-    needs: check
+  build-doc:
     runs-on:
       group: CI
       labels: [self-hosted, doc-build]
-    if: ${{ needs.check.outputs.changed_files_zh != '' || needs.check.outputs.changed_files_en != '' }}
-
     steps:
       - name: Get the latest document contents
         run: |
           set -e
-          #./.github/scripts/update_repo.sh ${{ env.DOC_WKC }}/${{ env.TD_REPO }} ${{ inputs.target_branch }} ${{ inputs.target_pr_number }}
           cd ${{ env.DOC_WKC }}/${{ env.TD_REPO }}
           git reset --hard
           git clean -f
           git remote prune origin
           git fetch
-          git checkout ${{ inputs.target_branch }}
+          git checkout ${{ github.event.pull_request.base.ref }}
           git pull >/dev/null
-          git fetch origin +refs/pull/${{ inputs.target_pr_number }}/merge
+          git fetch origin +refs/pull/${{ github.event.pull_request.number }}/merge
           git checkout -qf FETCH_HEAD
 
       - name: Build the chinese document
-        if: ${{ needs.check.outputs.changed_files_zh != '' }}
         run: |
           cd ${{ env.DOC_WKC }}/${{ env.ZH_DOC_REPO }}
           yarn ass local
           yarn build
 
       - name: Build the english document
-        if: ${{ needs.check.outputs.changed_files_en != '' }}
         run: |
           cd ${{ env.DOC_WKC }}/${{ env.EN_DOC_REPO }}
           yarn ass local
           yarn build
-
-    outputs:
-      changed_files_zh: ${{ needs.check.outputs.changed_files_zh }}
-      changed_files_en: ${{ needs.check.outputs.changed_files_en }}
-      changed_files_non_doc: ${{ needs.check.outputs.changed_files_non_doc }}
-      changed_files_non_tdgpt: ${{ needs.check.outputs.changed_files_non_tdgpt }}