Skip to content

Commit

Permalink
Source binaries for linux artifacts from docker images
Browse files Browse the repository at this point in the history
This gives us static binaries linked against musl for our release
artifacts, unifying our libc dependency for both docker and non-docker
and simplifying our build tooling.

Since artifact building is now fairly complicated and really only part
of the CI/CD pipeline, got rid of the Makefile target for it.

Fixes: #4346

Signed-off-by: Andrew Harding <[email protected]>
  • Loading branch information
azdagron committed Sep 13, 2023
1 parent 969e383 commit 62586ba
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 142 deletions.
22 changes: 12 additions & 10 deletions .github/workflows/pr_build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ jobs:
artifacts:
name: artifacts (linux)
runs-on: ubuntu-20.04
needs: [cache-deps]
needs: [cache-deps, images]
timeout-minutes: 30

permissions:
Expand All @@ -132,16 +132,16 @@ jobs:
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Load cached deps
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- name: Load cached build tools
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
- name: Install regctl
uses: regclient/actions/regctl-installer@b6614f5f56245066b533343a85f4109bdc38c8cc # main
- name: Download archived images
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
path: .build
key: ${{ runner.os }}-tools-${{ hashFiles('.go-version','Makefile') }}
name: images
path: .
- name: Expand archived images
run: |
tar xvf images.tar.gz
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive artifacts
Expand Down Expand Up @@ -543,6 +543,8 @@ jobs:
update: true
install: >-
git base-devel mingw-w64-x86_64-toolchain zip unzip
- name: Build binaries
run: make build
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive binaries
Expand Down
23 changes: 13 additions & 10 deletions .github/workflows/release_build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ jobs:
artifacts:
name: artifacts (linux)
runs-on: ubuntu-20.04
needs: [cache-deps]
needs: [cache-deps, images]
timeout-minutes: 30

permissions:
contents: read
Expand All @@ -125,16 +126,16 @@ jobs:
uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
go-version: ${{ env.GO_VERSION }}
- name: Load cached deps
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- name: Load cached build tools
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
- name: Install regctl
uses: regclient/actions/regctl-installer@b6614f5f56245066b533343a85f4109bdc38c8cc # main
- name: Download archived images
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
path: .build
key: ${{ runner.os }}-tools-${{ hashFiles('.go-version','Makefile') }}
name: images
path: .
- name: Expand archived images
run: |
tar xvf images.tar.gz
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive artifacts
Expand Down Expand Up @@ -531,6 +532,8 @@ jobs:
update: true
install: >-
git base-devel mingw-w64-x86_64-toolchain zip unzip
- name: Build binaries
run: make build
- name: Build artifacts
run: ./.github/workflows/scripts/build_artifacts.sh ${{ runner.os }}
- name: Archive binaries
Expand Down
42 changes: 21 additions & 21 deletions .github/workflows/scripts/build_artifacts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,32 +5,32 @@
set -e

usage() {
echo "usage: ${BASH_SOURCE[0]} <Linux|Windows|macOS>"
echo "usage: ${BASH_SOURCE[0]} <Linux|Windows>"
exit 1
}

[[ $# -eq 1 ]] || usage

os="$1"
declare -a supported_archs
if [[ "${os}" == "Linux" ]] || [[ "${os}" == "macOS" ]]; then
supported_archs=(amd64 arm64)
elif [[ "${os}" == "Windows" ]]; then
supported_archs=(amd64)
else
echo "unrecognized OS: ${os}"
usage
fi
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

export TAG=
if [[ "$GITHUB_REF" =~ ^refs/tags/v[0-9.]+$ ]]; then
# Strip off the leading "v" from the release tag. Release artifacts are
# named just with the version number (e.g. v0.9.3 tag produces
# spire-0.9.3-linux-x64.tar.gz).
TAG="${GITHUB_REF##refs/tags/v}"
# Strip off the leading "v" from the release tag. Release artifacts are
# named just with the version number (e.g. v0.9.3 tag produces
# spire-0.9.3-linux-x64.tar.gz).
TAG="${GITHUB_REF##refs/tags/v}"
fi

# Make references the $TAG environment variable set above
for arch in "${supported_archs[@]}"; do
GOARCH=$arch make artifact
done
[[ $# -eq 1 ]] || usage

os="$1"
case "${os}" in
Linux)
"${SCRIPTDIR}"/build_linux_artifacts.sh
;;
Windows)
"${SCRIPTDIR}"/build_windows_artifacts.sh
;;
*)
echo "Only artifacts for Linux and Windows are supported" 1>&2
usage
;;
esac
93 changes: 93 additions & 0 deletions .github/workflows/scripts/build_linux_artifacts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/bin/bash

set -e

REPODIR=$(git rev-parse --show-toplevel)

TAG=${TAG:-$(git log -n1 --pretty=%h)}
OUTDIR=${OUTDIR:-"${REPODIR}/artifacts"}

TARCMD=tar
if [[ $(uname -s) == "Darwin" ]]; then
# When building linux artifacts from darwin, gtar is required.
TARCMD="gtar"
fi

TAROPTS=("--owner=root" "--group=root")

TMPDIR=$(mktemp -d)
cleanup() {
rm -rf "${TMPDIR}"
}
trap cleanup EXIT


copy_binary_from_multiarch_tar() {
local arch=$1
local binary=$2
local destdir=$3

local srcpath="/opt/spire/bin/${binary}"
local destpath="${destdir}/${binary}"
local ocidir="ocidir://${TMPDIR}/${arch}/oci/${binary}"
local imagetar="${REPODIR}/${binary}-image.tar"
local platform="linux/${arch}"

echo "Importing multiarch image ${imagetar}..."
regctl image import "${ocidir}" "${imagetar}"

echo "Copying ${srcpath} for platform ${platform}..."
regctl image get-file "${ocidir}" "${srcpath}" "${destpath}" -p "${platform}"

# file does not retain permission bits, so fix up the executable bit.
chmod +x "${destpath}"
}

build_artifact() {
ARCH="$1"

ARTIFACT="${OUTDIR}/spire-${TAG}-linux-${ARCH}-musl.tar.gz"
CHECKSUM="${OUTDIR}/spire-${TAG}-linux-${ARCH}-musl_sha256sum.txt"

EXTRAS_ARTIFACT="${OUTDIR}/spire-extras-${TAG}-linux-${ARCH}-musl.tar.gz"
EXTRAS_CHECKSUM="${OUTDIR}/spire-extras-${TAG}-linux-${ARCH}-musl_sha256sum.txt"

TARDIR="${TMPDIR}/${ARCH}/tar"
mkdir -p "${TARDIR}"

STAGING="${TARDIR}"/spire/spire-${TAG}
EXTRAS_STAGING="${TARDIR}"/spire-extras/spire-extras-${TAG}
mkdir -p "${STAGING}" "${EXTRAS_STAGING}"

echo "Creating \"${ARTIFACT}\" and \"${EXTRAS_ARTIFACT}\""

# Copy in the contents under release/
cp -r "${REPODIR}"/release/posix/spire/* "${STAGING}"
cp -r "${REPODIR}"/release/posix/spire-extras/* "${EXTRAS_STAGING}"

# Copy in the LICENSE
cp "${REPODIR}"/LICENSE "${STAGING}"
cp "${REPODIR}"/LICENSE "${EXTRAS_STAGING}"

# Copy in the SPIRE binaries from the docker images:
# 1. import the image from the multiarch tarball into the OCI directory
mkdir -p "${STAGING}"/bin "${EXTRAS_STAGING}"/bin
copy_binary_from_multiarch_tar "$ARCH" "spire-server" "${STAGING}/bin"
copy_binary_from_multiarch_tar "$ARCH" "spire-agent" "${STAGING}/bin"
copy_binary_from_multiarch_tar "$ARCH" "oidc-discovery-provider" "${EXTRAS_STAGING}/bin"

mkdir -p "${OUTDIR}"

# Create the tarballs and checksums
(cd "${TARDIR}/spire"; ${TARCMD} -cvzf "${ARTIFACT}" "${TAROPTS[@]}" -- *)
(cd "${TARDIR}/spire-extras"; ${TARCMD} -cvzf "${EXTRAS_ARTIFACT}" "${TAROPTS[@]}" -- *)

(cd "$(dirname "${ARTIFACT}")"; shasum -a 256 "$(basename "${ARTIFACT}")" > "${CHECKSUM}" )
(cd "$(dirname "${EXTRAS_ARTIFACT}")"; shasum -a 256 "$(basename "${EXTRAS_ARTIFACT}")" > "${EXTRAS_CHECKSUM}" )
}

command -v regctl >/dev/null 2>&1 || { echo -e "The regctl cli is required to run this script." >&2 ; exit 1; }
command -v "${TARCMD}" >/dev/null 2>&1 || { echo -e "The ${TARCMD} command is required to run this script." >&2 ; exit 1; }

build_artifact amd64
build_artifact arm64
51 changes: 51 additions & 0 deletions .github/workflows/scripts/build_windows_artifacts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/bash

set -e

REPODIR=$(git rev-parse --show-toplevel)
BINDIR="${REPODIR}/bin"

TAG=${TAG:-$(git log -n1 --pretty=%h)}
OUTDIR=${OUTDIR:-"${REPODIR}/artifacts"}

ARCH=amd64

ARTIFACT="${OUTDIR}/spire-${TAG}-windows-${ARCH}.zip"
CHECKSUM="${OUTDIR}/spire-${TAG}-windows-${ARCH}_sha256sum.txt"

EXTRAS_ARTIFACT="${OUTDIR}/spire-extras-${TAG}-windows-${ARCH}.zip"
EXTRAS_CHECKSUM="${OUTDIR}/spire-extras-${TAG}-windows-${ARCH}_sha256sum.txt"

TMPDIR=$(mktemp -d)
cleanup() {
rm -rf "${TMPDIR}"
}
trap cleanup EXIT

STAGING="${TMPDIR}"/spire/spire-${TAG}
EXTRAS_STAGING="${TMPDIR}"/spire-extras/spire-extras-${TAG}
mkdir -p "${STAGING}" "${EXTRAS_STAGING}"

echo "Creating \"${ARTIFACT}\" and \"${EXTRAS_ARTIFACT}\""

# Copy in the contents under release/
cp -r "${REPODIR}"/release/windows/spire/* "${STAGING}"
cp -r "${REPODIR}"/release/windows/spire-extras/* "${EXTRAS_STAGING}"

# Copy in the LICENSE
cp "${REPODIR}"/LICENSE "${STAGING}"
cp "${REPODIR}"/LICENSE "${EXTRAS_STAGING}"

# Copy in the SPIRE binaries
mkdir -p "${STAGING}"/bin "${EXTRAS_STAGING}"/bin
cp "${BINDIR}"/spire-server.exe "${STAGING}"/bin
cp "${BINDIR}"/spire-agent.exe "${STAGING}"/bin
cp "${BINDIR}"/oidc-discovery-provider.exe "${EXTRAS_STAGING}"/bin

mkdir -p "${OUTDIR}"

(cd "${TMPDIR}/spire"; zip -rv "${ARTIFACT}" -- *)
(cd "${TMPDIR}/spire-extras"; zip -rv "${EXTRAS_ARTIFACT}" -- *)

(cd "$(dirname "${ARTIFACT}")"; CertUtil -hashfile "$(basename "${ARTIFACT}")" SHA256 > "${CHECKSUM}")
(cd "$(dirname "${EXTRAS_ARTIFACT}")"; CertUtil -hashfile "$(basename "${EXTRAS_ARTIFACT}")" SHA256 > "${EXTRAS_CHECKSUM}")
10 changes: 0 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ help:
@echo
@echo "$(bold)Build:$(reset)"
@echo " $(cyan)build$(reset) - build all SPIRE binaries (default)"
@echo " $(cyan)artifact$(reset) - build SPIRE tarball artifact"
@echo
@echo "$(bold)Test:$(reset)"
@echo " $(cyan)test$(reset) - run unit tests"
Expand Down Expand Up @@ -318,15 +317,6 @@ endif
integration-windows:
$(E)./test/integration/test-windows.sh $(SUITES)

#############################################################################
# Build Artifact
#############################################################################

.PHONY: artifact

artifact: build
$(E)OUTDIR="$(OUTDIR)" TAG="$(TAG)" ./script/build-artifact.sh

#############################################################################
# Docker Images
#############################################################################
Expand Down
Loading

0 comments on commit 62586ba

Please sign in to comment.