fix: release build scripts (backport #1286) (#1373)

* fix: release build scripts (#1286)

* update all builders

* rbuilder images are prefixed by osmolabs

* update makefile

* use root instead of builder user and other minor changes

* correct lib path in Makefile

* LINK_STATICALLY in Makefile

* docs for builder

* update README

* changelog entry

* remove make clean from .build.sh

* improve readme

* remove spurious whitespace in Makefile

* fix formatting for build-reproducible

* update cosmwasm path in makefile

* copy from /home/builder in build-reproducible

* change wasm file pemissions to non-root

(cherry picked from commit 2029415)

# Conflicts:
#	CHANGELOG.md

* fix: release build scripts (#1286)

* update all builders

* rbuilder images are prefixed by osmolabs

* update makefile

* use root instead of builder user and other minor changes

* correct lib path in Makefile

* LINK_STATICALLY in Makefile

* docs for builder

* update README

* changelog entry

* remove make clean from .build.sh

* improve readme

* remove spurious whitespace in Makefile

* fix formatting for build-reproducible

* update cosmwasm path in makefile

* copy from /home/builder in build-reproducible

* change wasm file pemissions to non-root

* fix changelog

Co-authored-by: Roman <[email protected]>
Co-authored-by: Adam Tucker <[email protected]>

.build.sh

@@ -1,4 +1,3 @@
-
 #!/bin/bash
 
 set -ue
@@ -12,7 +11,7 @@ set -ue
 # - DEBUG
 
 # Source builder's functions library
-. /usr/local/share/tendermint/buildlib.sh
+. /usr/local/share/osmosis/buildlib.sh
 
 # These variables are now available
 # - BASEDIR
@@ -25,14 +24,15 @@ for platform in ${TARGET_PLATFORMS} ; do
     # cases except when the target platform is 'windows'.
     setup_build_env_for_platform "${platform}"
 
-    make clean
     echo Building for $(go env GOOS)/$(go env GOARCH) >&2
     GOROOT_FINAL="$(go env GOROOT)" \
     make build \
         LDFLAGS=-buildid=${VERSION} \
         VERSION=${VERSION} \
         COMMIT=${COMMIT} \
-        LEDGER_ENABLED=${LEDGER_ENABLED}
+        LEDGER_ENABLED=${LEDGER_ENABLED} \
+        LINK_STATICALLY=true \
+        BUILD_TAGS=muslc
     mv ./build/${APP}${OS_FILE_EXT} ${OUTDIR}/${APP}-${VERSION}-$(go env GOOS)-$(go env GOARCH)${OS_FILE_EXT}
 
     # This function restore the build environment variables to their

Makefile

@@ -66,6 +66,9 @@ endif
 ifeq (,$(findstring nostrip,$(OSMOSIS_BUILD_OPTIONS)))
   ldflags += -w -s
 endif
+ifeq ($(LINK_STATICALLY),true)
+	ldflags += -linkmode=external -extldflags "-L/usr/local/lib/ -lwasmvm_muslc -Wl,-z,muldefs -static"
+endif
 ldflags += $(LDFLAGS)
 ldflags := $(strip $(ldflags))
 
@@ -97,12 +100,12 @@ $(BUILDDIR)/:
 build-reproducible: go.sum
 	$(DOCKER) rm latest-build || true
 	$(DOCKER) run --volume=$(CURDIR):/sources:ro \
-        --env TARGET_PLATFORMS='linux/amd64 darwin/amd64 linux/arm64 windows/amd64' \
-        --env APP=osmosisd \
-        --env VERSION=$(VERSION) \
-        --env COMMIT=$(COMMIT) \
-        --env LEDGER_ENABLED=$(LEDGER_ENABLED) \
-        --name latest-build cosmossdk/rbuilder:latest
+	--env TARGET_PLATFORMS='linux/amd64' \
+	--env APP=osmosisd \
+	--env VERSION=$(VERSION) \
+	--env COMMIT=$(COMMIT) \
+	--env LEDGER_ENABLED=$(LEDGER_ENABLED) \
+	--name latest-build osmolabs/rbuilder:latest
 	$(DOCKER) cp -a latest-build:/home/builder/artifacts/ $(CURDIR)/
 
 build-linux: go.sum
@@ -126,7 +129,7 @@ draw-deps:
 	@goviz -i ./cmd/osmosisd -d 2 | dot -Tpng -o dependency-graph.png
 
 clean:
-	rm -rf $(BUILDDIR)/ artifacts/
+	rm -rf $(CURDIR)/artifacts/
 
 distclean: clean
 	rm -rf vendor/

contrib/images/Makefile

@@ -4,6 +4,6 @@ simd-env:
 	docker build --build-arg UID=$(shell id -u) --build-arg GID=$(shell id -g) --tag cosmossdk/simd-env simd-env
 
 rbuilder:
-	docker build --tag cosmossdk/rbuilder rbuilder
+	docker build --tag osmolabs/rbuilder rbuilder
 
 .PHONY: all simd-env rbuilder

contrib/images/rbuilder/Dockerfile

@@ -1,16 +1,37 @@
-FROM golang:1.15.2-alpine3.12
+
+FROM golang:1.18-bullseye
+
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get --no-install-recommends -y install \
     pciutils build-essential git wget \
     lsb-release dpkg-dev curl bsdmainutils fakeroot
-RUN mkdir -p /usr/local/share/cosmos-sdk/
-COPY buildlib.sh /usr/local/share/cosmos-sdk/
+RUN mkdir -p /usr/local/share/osmosis/
+
+# Deploy the shell functions library.
+COPY buildlib.sh /usr/local/share/osmosis/
+
+# Create the 'builder' user.
 RUN useradd -ms /bin/bash -U builder
 ARG APP
 ARG DEBUG
-ENV APP ${APP:-cosmos-sdk}
-ENV DEBUG ${DEBUG} VERSION unknown COMMIT unknown LEDGER_ENABLE true
+ARG TARGET_PLATFORMS
+ENV APP ${APP:-app}
+ENV DEBUG ${DEBUG}
+ENV VERSION unknown
+ENV COMMIT unknown
+ENV LEDGER_ENABLE true
+ENV TARGET_PLATFORMS ${TARGET_PLATFORMS:-linux/amd64}
+ENV BUILD_SCRIPT ${BUILD_SCRIPT:-/sources/.build.sh}
+
+# From https://github.com/CosmWasm/wasmd/blob/master/Dockerfile
+# For more details see https://github.com/CosmWasm/wasmvm#builds-of-libwasmvm 
+ADD https://github.com/CosmWasm/wasmvm/releases/download/v1.0.0-beta7/libwasmvm_muslc.a /usr/local/lib/libwasmvm_muslc.a
+RUN chown builder /usr/local/lib/libwasmvm_muslc.a 
+RUN sha256sum /usr/local/lib/libwasmvm_muslc.a | grep d0152067a5609bfdfb3f0d5d6c0f2760f79d5f2cd7fd8513cafa9932d22eb350
+
+# Drop root privileges.
 USER builder:builder
 WORKDIR /sources
-VOLUME [ "/sources" ]
-ENTRYPOINT [ "/sources/build.sh" ]
+
+# Run the application's build.sh.
+ENTRYPOINT [ "/bin/bash", "-c", "${BUILD_SCRIPT}" ]

contrib/images/rbuilder/README.md

@@ -0,0 +1,44 @@
+# Reproducible Build System
+
+This image and scripts are meant to provide a minimal deterministic
+build system for the Osmosis application.
+
+It was created by referencing Tendermint's [implementation](https://github.com/tendermint/images/blob/cf0d1a9f3731e30540bbfa36a36d13e4dcccf5eb/rbuilder/README.md)
+
+# Requirements And Usage
+
+The Osmosis repository must include a`.build.sh` executable file 
+in the root folder meant to drive the build process.
+
+The build's outputs are produced in the top-level `artifacts` directory.
+
+## Building the Image Locally
+
+```
+cd ./contrib/images
+
+make rbuilder
+```
+
+This creates the `rbuilder` image. To run a container of this image locally and build the binaries:
+```
+cd <osmosis root>
+
+make build-reproducible
+```
+
+This spins up an `rbuilder` container with a volume installed to the
+root of the repository. This way, the builder has access to the `.build.sh`file and is
+able to execute it.
+
+## Wasmvm dependency
+
+Currently, only `linux/amd64` is supported. Adding more support is blocked by our dependency on wasmvm.
+
+The support of some platforms is already added in new versions of wasmvm.
+Follow the release log for more detaisl when updating our builder:
+https://github.com/CosmWasm/wasmvm/releases
+
+Once wasmvm is upgraded, more platforms may be built by changing
+`TARGET_PLATFORMS` environment variable in `build-reproducible`
+Makefile step. 

contrib/images/rbuilder/buildlib.sh

@@ -20,29 +20,29 @@ f_setup_pristine_src_dir() {
     go mod download
 }
 
-f_build_archs() {
-    local l_os
-
-    l_os=$1
-
-    case "${l_os}" in
-    darwin | windows)
-        echo 'amd64'
-        ;;
-    linux)
-        echo 'amd64 arm64'
-        ;;
-    *)
-        echo "unknown OS -- ${l_os}" >&2
-        return 1
-    esac
+f_exe_file_ext() {
+   [ $(go env GOOS) = windows ] && printf '%s' '.exe' || printf ''
 }
 
-f_binary_file_ext() {
-   [ $1 = windows ] && printf '%s' '.exe' || printf ''
+setup_build_env_for_platform() {
+    local l_platform=$1
+
+    g_old_GOOS="$(go env GOOS)"
+    g_old_GOARCH="$(go env GOARCH)"
+    g_old_OS_FILE_EXT="${OS_FILE_EXT}"
+
+    go env -w GOOS="${l_platform%%/*}"
+    go env -w GOARCH="${l_platform##*/}"
+    OS_FILE_EXT="$(f_exe_file_ext)"
+}
+
+restore_build_env() {
+    go env -w GOOS="${g_old_GOOS}"
+    go env -w GOARCH="${g_old_GOARCH}"
+    OS_FILE_EXT="${g_old_OS_FILE_EXT}"
 }
 
-f_generate_build_report() {
+generate_build_report() {
     local l_tempfile
 
     l_tempfile="$(mktemp)"
@@ -63,6 +63,7 @@ EOF
 
 [ "x${DEBUG}" = "x" ] || set -x
 
+OS_FILE_EXT=''
 BASEDIR="$(mktemp -d)"
 OUTDIR=$HOME/artifacts
 rm -rfv ${OUTDIR}/