Add GHA workflow for pushing to GCP #28
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: RPM Build and Test | |
| env: | |
| # TODO: we really need to define a list of supported versions (ideally it's no more than 2) | |
| # build is done on the lowest version and test on the highest with a "sanity test" | |
| # stage done on all versions in the list ecept the highest | |
| EL8_BUILD_VERSION: 8 | |
| EL8_VERSION: 8 | |
| EL9_BUILD_VERSION: 9 | |
| EL9_VERSION: 9 | |
| LEAP15_VERSION: 15.5 | |
| # temporarily disable EL9 builds | |
| CP_SKIP_BUILD_EL9_RPM: true | |
| CP_SKIP_BUILD_LEAP15_RPM: true | |
| GCS_BUCKET: github_ci_artifacts | |
| GCS_BUCKET_MOUNT_PT: /tmp/gcp_artifacts | |
| ARTIFACTORY_URL: | |
| JENKINS_URL: file:///tmp/gcp_artifacts/ | |
| REPO_PATH: /tmp/gcp_artifacts/job_repos/daos-stack/job/libfabric/job/PR-${{ github.event.pull_request.number }}/ | |
| REPOSITORY_URL: | |
| ARTIFACTS_URL: file:///tmp/gcp_artifacts/job_repos/ | |
| REPO_FILE_URL: | |
| PROJECT_ID: daos-github-ci | |
| WIP_PROVIDER: projects/192572342707/locations/global/workloadIdentityPools/github/providers/daos | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr-repos: | |
| description: 'Any PR-repos that you want included in this build' | |
| required: false | |
| pull_request: | |
| concurrency: | |
| group: rpm-build-and-test-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash --noprofile --norc -ueo pipefail {0} | |
| permissions: {} | |
| jobs: | |
| Create-symlinks: | |
| # you might think this is an odd place to do this and it should be done as a result of the | |
| # build and/or testing stages and ideally you'd be right. | |
| # the problem with that is that there is no way to get the success/fail result of individual | |
| # axes of matrix jobs so there is no way to query them at the end and see their composite | |
| # results. | |
| # instead, the final result of the Build-RPM job, for example is a last-one-complete wins. | |
| # so for example, if the el9 axis fails quickly and then the el8 axis succeeds afterward the | |
| # resulting job state is success. | |
| # instead we assume success at the beginning and then let any axis that fails remove the | |
| # lastSuccessfulBuild link if it fails | |
| name: Create lastBuild and lastSuccessfulBuild symlinks | |
| permissions: | |
| statuses: write | |
| contents: read | |
| id-token: write | |
| runs-on: [self-hosted, gcp] | |
| steps: | |
| # we are required to checkout the code to have auth save the token we need for GCSFUSE | |
| # per https://github.com/google-github-actions/auth#inputs-miscellaneous | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Authenticate to GCP | |
| id: gcp-authentication | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| project_id: ${{ env.PROJECT_ID }} | |
| workload_identity_provider: ${{ env.WIP_PROVIDER }} | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| - name: Set up GCSFUSE | |
| run: | | |
| sudo fusermount -u ${GCS_BUCKET_MOUNT_PT} && sudo rm -fr /tmp/gcp_artifacts || true | |
| mkdir -p "${GCS_BUCKET_MOUNT_PT}" | |
| #weird hack needed to prevent gcsfuse from authentication failure | |
| sudo -u $(whoami) gcsfuse --implicit-dirs "${GCS_BUCKET}" "${GCS_BUCKET_MOUNT_PT}" | |
| - name: Create lastBuild and lastSuccessfulBuild symlinks | |
| run: mkdir -p ${REPO_PATH}; | |
| rm -f ${REPO_PATH}last{,Successful}Build; | |
| ln -s ${{ github.run_number }} ${REPO_PATH}lastBuild; | |
| ln -s ${{ github.run_number }} ${REPO_PATH}lastSuccessfulBuild | |
| Calc-rpm-build-matrix: | |
| name: Calculate RPM Build Matrix | |
| runs-on: [self-hosted, gcp] | |
| needs: [Create-symlinks] | |
| outputs: | |
| matrix: ${{ steps.matrix.outputs.text }} | |
| steps: | |
| - name: Calculate RPM Build Matrix | |
| id: matrix | |
| run: | # do not use the non-| format for this script | |
| l=() | |
| trap 'echo "text=[$(IFS=","; echo "${l[*]}")]" >> $GITHUB_OUTPUT' EXIT | |
| if ${CP_SKIP_BUILD:-false}; then | |
| exit 0 | |
| fi | |
| if ! ${CP_SKIP_BUILD_EL8_RPM:-false}; then | |
| l+=('"el8"') | |
| fi | |
| if ! ${CP_SKIP_BUILD_EL9_RPM:-false}; then | |
| l+=('"el9"') | |
| fi | |
| if ${{ github.event_name == 'push' }} || | |
| (${{ github.event_name == 'pull_request' }} && | |
| ! ${CP_SKIP_BUILD_LEAP15_RPM:-false}); then | |
| l+=('"leap15"') | |
| fi | |
| Build-RPM: | |
| name: Build RPM | |
| permissions: | |
| statuses: write | |
| contents: read | |
| id-token: write | |
| runs-on: [self-hosted, gcp] | |
| needs: [Create-symlinks, Calc-rpm-build-matrix] | |
| if: needs.Create-symlinks.result == 'success' && | |
| ((!cancelled()) || success() || failure()) | |
| strategy: | |
| matrix: | |
| distro: ${{ fromJSON(needs.Calc-rpm-build-matrix.outputs.matrix) }} | |
| fail-fast: false | |
| env: | |
| DAOS_EMAIL: [email protected] | |
| DAOS_FULLNAME: daos-stack | |
| DISTRO: ${{ matrix.distro }} | |
| DISTRO_REPOS: disabled | |
| DOCKER_BUILDKIT: 0 | |
| MOCK_OPTIONS: --uniqueext=${{ github.run_id }} | |
| PR_NUM: ${{ github.event.pull_request.number }} | |
| # TODO -- this should be on stable, backedup storage, not /scratch | |
| # yamllint disable-line rule:line-length | |
| RUN_ID: ${{ github.run_id }} | |
| TARGET: ${{ matrix.distro }} | |
| # keep VS Code's GHA linting happy | |
| STAGE_NAME: | |
| DISTRO_NAME: | |
| DISTRO_VERSION: | |
| CP_LEAP15_VERSION: | |
| COMMIT_STATUS_DISTRO_VERSION: | |
| FVERSION: | |
| steps: | |
| - name: Set variables | |
| run: | | |
| FVERSION="38" | |
| case ${{ matrix.distro }} in | |
| 'el8') | |
| CHROOT_NAME="rocky+epel-8-x86_64" | |
| DISTRO_NAME="EL" | |
| DISTRO_VERSION="${{ env.EL8_BUILD_VERSION }}" | |
| COMMIT_STATUS_DISTRO_VERSION="8" | |
| ;; | |
| 'el9') | |
| CHROOT_NAME="rocky+epel-9-x86_64" | |
| DISTRO_NAME="EL" | |
| DISTRO_VERSION="${{ env.EL9_BUILD_VERSION }}" | |
| ;; | |
| 'leap15') | |
| CHROOT_NAME="opensuse-leap-${{ env.CP_LEAP15_VERSION && | |
| env.CP_LEAP15_VERSION || | |
| env.LEAP15_VERSION }}-x86_64" | |
| DISTRO_NAME="Leap" | |
| DISTRO_VERSION="${{ env.CP_LEAP15_VERSION && | |
| env.CP_LEAP15_VERSION || env.LEAP15_VERSION }}" | |
| ;; | |
| esac | |
| echo "CHROOT_NAME=$CHROOT_NAME" >> $GITHUB_ENV | |
| echo "DISTRO_NAME=$DISTRO_NAME" >> $GITHUB_ENV | |
| echo "DISTRO_VERSION=$DISTRO_VERSION" >> $GITHUB_ENV | |
| echo "BUILD_CHROOT=/var/lib/mock/$CHROOT_NAME-${{ github.run_id }}/" >> $GITHUB_ENV | |
| echo "STAGE_NAME=Build GCP RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV | |
| echo "FVERSION=$FVERSION" >> $GITHUB_ENV | |
| echo "COMMIT_STATUS_DISTRO_VERSION=$COMMIT_STATUS_DISTRO_VERSION" >> $GITHUB_ENV | |
| echo "REPO_BUILD_PATH=${REPO_PATH}${{ github.run_number }}/artifact/artifacts/$TARGET" >> $GITHUB_ENV | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Build RPM Docker image | |
| id: build-rpm-docker-image | |
| continue-on-error: true | |
| run: docker build --file packaging/Dockerfile.mockbuild | |
| --build-arg CACHEBUST=$(date +%s%3N) | |
| --build-arg CB0=$(date +%V) | |
| --build-arg REPO_FILE_URL=$REPO_FILE_URL | |
| --build-arg UID=$(id -u) | |
| --build-arg FVERSION=${{ env.FVERSION }} | |
| --tag mock-build | |
| . | |
| - name: Build RPM | |
| id: build-rpm | |
| continue-on-error: true | |
| # yamllint disable rule:line-length | |
| run: rm -rf mock_result; | |
| mkdir -p mock_result; | |
| docker run --name mock-build-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }} | |
| --user build | |
| -v "$PWD":"$PWD" -w "$PWD" | |
| -v "$PWD"/mock_result:/var/lib/mock/$CHROOT_NAME/result | |
| --privileged=true | |
| -e DAOS_FULLNAME="$DAOS_FULLNAME" | |
| -e DAOS_EMAIL="$DAOS_EMAIL" | |
| -e DISTRO_VERSION="$DISTRO_VERSION" | |
| -e STAGE_NAME="$STAGE_NAME" | |
| -e CHROOT_NAME="$CHROOT_NAME" | |
| -e ARTIFACTORY_URL="$ARTIFACTORY_URL" | |
| -e REPO_FILE_URL="$REPO_FILE_URL" | |
| -e JENKINS_URL="$JENKINS_URL" | |
| -e TARGET="$TARGET" | |
| mock-build make chrootbuild | |
| # yamllint enable rule:line-length | |
| - name: Build RPM failure log | |
| id: build-rpm-fail-log | |
| continue-on-error: true | |
| if: steps.build-rpm.outcome != 'success' | |
| run: cat mock_result/root.log; | |
| cat mock_result/build.log | |
| - name: Save RPM build logs | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.STAGE_NAME }} logs | |
| path: | | |
| mock_result/root.log | |
| mock_result/build.log | |
| - name: Create repo | |
| id: create-repo | |
| if: steps.build-rpm.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| mkdir -p "${REPO_BUILD_PATH}" | |
| cp -a mock_result/*.rpm "${REPO_BUILD_PATH}" | |
| cd "${REPO_BUILD_PATH}" | |
| createrepo . | |
| - name: Test repo | |
| id: test-repo | |
| if: steps.create-repo.outcome == 'success' | |
| continue-on-error: true | |
| run: dnf --disablerepo=\* --repofrompath | |
| testrepo,"${REPO_BUILD_PATH}" | |
| repoquery -a | |
| - name: Authenticate to GCP | |
| id: gcp-authentication | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| project_id: ${{ env.PROJECT_ID }} | |
| workload_identity_provider: ${{ env.WIP_PROVIDER }} | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v2" | |
| - name: Set up GCSFUSE | |
| run: | | |
| sudo fusermount -u ${GCS_BUCKET_MOUNT_PT} && sudo rm -fr /tmp/gcp_artifacts || true | |
| mkdir -p "${GCS_BUCKET_MOUNT_PT}" | |
| sudo -u ext_mlawsonca_google_com gcsfuse --implicit-dirs "${GCS_BUCKET}" "${GCS_BUCKET_MOUNT_PT}" | |
| - name: Remove lastSuccessfulBuild link and exit failure | |
| if: steps.test-repo.outcome != 'success' | |
| run: rm -f ${REPO_PATH}lastSuccessfulBuild; | |
| exit 1 | |
| - name: Publish RPMs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.DISTRO_NAME }} ${{ env.DISTRO_VERSION }} GCP RPM repository | |
| path: ${{ env.REPO_PATH}}${{ github.run_number }}/artifact/artifacts/${{ env.TARGET }} | |
| - name: Update commit status | |
| uses: ouzi-dev/commit-status-updater@v2 | |
| with: | |
| # yamllint disable-line rule:line-length | |
| name: 'build/Build RPM on ${{ env.DISTRO_NAME }} ${{ env.COMMIT_STATUS_DISTRO_VERSION && env.COMMIT_STATUS_DISTRO_VERSION || env.DISTRO_VERSION }}' | |
| status: "${{ job.status }}" |