gcp-rpm-build-and-test.yml

name: RPM Build and Test

env:
  # TODO: we really need to define a list of supported versions (ideally it's no more than 2)
  #       build is done on the lowest version and test on the highest with a "sanity test"
  #       stage done on all versions in the list ecept the highest
  EL8_BUILD_VERSION: 8
  EL8_VERSION: 8
  EL9_BUILD_VERSION: 9
  EL9_VERSION: 9
  LEAP15_VERSION: 15.5
  # temporarily disable EL9 builds
  CP_SKIP_BUILD_EL9_RPM: true
  CP_SKIP_BUILD_LEAP15_RPM: true
  GCS_BUCKET: github_ci_artifacts
  GCS_BUCKET_MOUNT_PT: /tmp/gcp_artifacts
  ARTIFACTORY_URL:
  JENKINS_URL: file:///tmp/gcp_artifacts/
  REPO_PATH: file:///tmp/gcp_artifacts/job_repos/daos-stack/job/libfabric/job/PR-${{ github.event.pull_request.number }}/
  REPOSITORY_URL:
  ARTIFACTS_URL: file:///tmp/gcp_artifacts/job_repos/
  REPO_FILE_URL:
  PROJECT_ID: daos-github-ci
  WIP_PROVIDER: projects/192572342707/locations/global/workloadIdentityPools/github/providers/daos

on:
  workflow_dispatch:
    inputs:
      pr-repos:
        description: 'Any PR-repos that you want included in this build'
        required: false
  pull_request:

concurrency:
  group: rpm-build-and-test-${{ github.head_ref  || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash --noprofile --norc -ueo pipefail {0}

permissions: {}

jobs:
  Create-symlinks:
    # you might think this is an odd place to do this and it should be done as a result of the
    # build and/or testing stages and ideally you'd be right.
    # the problem with that is that there is no way to get the success/fail result of individual
    # axes of matrix jobs so there is no way to query them at the end and see their composite
    # results.
    # instead, the final result of the Build-RPM job, for example is a last-one-complete wins.
    # so for example, if the el9 axis fails quickly and then the el8 axis succeeds afterward the
    # resulting job state is success.
    # instead we assume success at the beginning and then let any axis that fails remove the
    # lastSuccessfulBuild link if it fails
    name: Create lastBuild and lastSuccessfulBuild symlinks
    permissions:
      statuses: write
      contents: read
      id-token: write
    runs-on: [self-hosted, gcp]
    steps:
      # we are required to checkout the code to have auth save the token we need for GCSFUSE
      # per https://github.com/google-github-actions/auth#inputs-miscellaneous
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Authenticate to GCP
        id: gcp-authentication
        uses: 'google-github-actions/auth@v2'
        with:
          project_id: ${{ env.PROJECT_ID }}
          workload_identity_provider: ${{ env.WIP_PROVIDER }}
      - name: "Set up Cloud SDK"
        uses: "google-github-actions/setup-gcloud@v2"
      - name: Set up GCSFUSE
        run: |
              sudo fusermount -u ${GCS_BUCKET_MOUNT_PT} && sudo rm -fr /tmp/gcp_artifacts || true
              mkdir -p "${GCS_BUCKET_MOUNT_PT}"
              gcsfuse --implicit-dirs "${GCS_BUCKET}" "${GCS_BUCKET_MOUNT_PT}"
      - name: Create lastBuild and lastSuccessfulBuild symlinks
        run: mkdir -p ${REPO_PATH};
               rm -f ${REPO_PATH}last{,Successful}Build;
               ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild;
               ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild

  Calc-rpm-build-matrix:
    name: Calculate RPM Build Matrix
    runs-on: [self-hosted, gcp]
    needs: [Create-symlinks]
    outputs:
      matrix: ${{ steps.matrix.outputs.text }}
    steps:
      - name: Calculate RPM Build Matrix
        id: matrix
        run: |  # do not use the non-| format for this script
             l=()
             trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT' EXIT
             if ${CP_SKIP_BUILD:-false}; then
                 exit 0
             fi
             if ! ${CP_SKIP_BUILD_EL8_RPM:-false}; then
                 l+=('"el8"')
             fi
             if ! ${CP_SKIP_BUILD_EL9_RPM:-false}; then
                 l+=('"el9"')
             fi
             if ${{ github.event_name == 'push' }} ||
                (${{ github.event_name == 'pull_request' }} &&
                ! ${CP_SKIP_BUILD_LEAP15_RPM:-false}); then
                 l+=('"leap15"')
             fi

  Build-RPM:
    name: Build RPM
    permissions:
      statuses: write
      contents: read
      id-token: write
    runs-on: [self-hosted, gcp]
    needs: [Create-symlinks, Calc-rpm-build-matrix]
    if: needs.Create-symlinks.result == 'success' &&
        ((!cancelled()) || success() || failure())
    strategy:
      matrix:
        distro: ${{ fromJSON(needs.Calc-rpm-build-matrix.outputs.matrix) }}
      fail-fast: false
    env:
      DAOS_EMAIL: brian.murrell@intel.com
      DAOS_FULLNAME: daos-stack
      DISTRO: ${{ matrix.distro }}
      DISTRO_REPOS: disabled
      DOCKER_BUILDKIT: 0
      MOCK_OPTIONS: --uniqueext=${{ github.run_id }}
      PR_NUM: ${{ github.event.pull_request.number }}
      # TODO -- this should be on stable, backedup storage, not /scratch
      # yamllint disable-line rule:line-length
      RUN_ID: ${{ github.run_id }}
      TARGET: ${{ matrix.distro }}
      # keep VS Code's GHA linting happy
      STAGE_NAME:
      DISTRO_NAME:
      DISTRO_VERSION:
      CP_LEAP15_VERSION:
      COMMIT_STATUS_DISTRO_VERSION:
      FVERSION:
    steps:
      - name: Set variables
        run: |
            FVERSION="38"
            case ${{ matrix.distro }} in
                'el8')
                    CHROOT_NAME="rocky+epel-8-x86_64"
                    DISTRO_NAME="EL"
                    DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}"
                    COMMIT_STATUS_DISTRO_VERSION="8"
                    ;;
                'el9')
                    CHROOT_NAME="rocky+epel-9-x86_64"
                    DISTRO_NAME="EL"
                    DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}"
                    ;;
                'leap15')
                    CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION &&
                                                   env.CP_LEAP15_VERSION ||
                                                   env.LEAP15_VERSION }}-x86_64"
                    DISTRO_NAME="Leap"
                    DISTRO_VERSION="${{ env.CP_LEAP15_VERSION &&
                                        env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}"
                    ;;
            esac
            echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV
            echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV
            echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV
            echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV
            echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV
            echo "FVERSION=$FVERSION" >> $GITHUB_ENV
            echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Build RPM Docker image
        id: build-rpm-docker-image
        continue-on-error: true
        run: docker build --file packaging/Dockerfile.mockbuild
                          --build-arg CACHEBUST=$(date +%s%3N)
                          --build-arg CB0=$(date +%V)
                          --build-arg REPO_FILE_URL=$REPO_FILE_URL
                          --build-arg UID=$(id -u)
                          --build-arg FVERSION=${{ env.FVERSION }}
                          --tag mock-build
                          .
      - name: Build RPM
        id: build-rpm
        continue-on-error: true
        # yamllint disable rule:line-length
        run: rm -rf mock_result;
             mkdir -p mock_result;
             docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}
                        --user build
                        -v "$PWD":"$PWD" -w "$PWD"
                        -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result
                        --privileged=true
                        -e DAOS_FULLNAME="$DAOS_FULLNAME"
                        -e DAOS_EMAIL="$DAOS_EMAIL"
                        -e DISTRO_VERSION="$DISTRO_VERSION"
                        -e STAGE_NAME="$STAGE_NAME"
                        -e CHROOT_NAME="$CHROOT_NAME"
                        -e ARTIFACTORY_URL="$ARTIFACTORY_URL"
                        -e REPO_FILE_URL="$REPO_FILE_URL"
                        -e JENKINS_URL="$JENKINS_URL"
                        -e TARGET="$TARGET"
                        mock-build make chrootbuild
        # yamllint enable rule:line-length
      - name: Build RPM failure log
        id: build-rpm-fail-log
        continue-on-error: true
        if: steps.build-rpm.outcome != 'success'
        run: cat mock_result/root.log;
             cat mock_result/build.log
      - name: Save RPM build logs
        continue-on-error: true
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.STAGE_NAME }} logs
          path: |
            mock_result/root.log
            mock_result/build.log
      - name: Create repo
        id: create-repo
        if: steps.build-rpm.outcome == 'success'
        continue-on-error: true
        run: CHROOT_NAME=$CHROOT_NAME ci/rpm/create_repo.sh
      - name: Test repo
        id: test-repo
        if: steps.create-repo.outcome == 'success'
        continue-on-error: true
        run: . ci/gha_functions.sh;
             dnf --disablerepo=\* --repofrompath
                 testrepo,file://${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET
                 repoquery -a
      - name: Authenticate to GCP
        id: gcp-authentication
        uses: 'google-github-actions/auth@v2'
        with:
          project_id: ${{ env.PROJECT_ID }}
          workload_identity_provider: ${{ env.WIP_PROVIDER }}
      - name: "Set up Cloud SDK"
        uses: "google-github-actions/setup-gcloud@v2"
      - name: Set up GCSFUSE
        run: |
              sudo fusermount -u ${GCS_BUCKET_MOUNT_PT} && sudo rm -fr /tmp/gcp_artifacts || true
              mkdir -p "${GCS_BUCKET_MOUNT_PT}"
              gcsfuse --implicit-dirs "${GCS_BUCKET}" "${GCS_BUCKET_MOUNT_PT}"
      - name: Remove lastSuccessfulBuild link and exit failure
        if: steps.test-repo.outcome != 'success'
        run: rm -f ${REPO_PATH}lastSuccessfulBuild;
             exit 1
      - name: Publish RPMs
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} GCP RPM repository
          path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }}
      - name: Update commit status
        uses: ouzi-dev/commit-status-updater@v2
        with:
          # yamllint disable-line rule:line-length
          name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}'
          status: "${{ job.status }}"