chore: bootstrap improvements. (#4711)

* `l1-contracts` cli cache bootstrappable.
* `noir-contracts` cli cache bootstrappable.
* `bootstrap.sh` will always attempt to pull from cli cache if the user
has docker and aws-credentials.
* `bootstrap.sh full` will skip trying to use the cache.
* `avm-transpiler` builds on focal for better compatability.
* New script `remove_old_images` removes all docker images for a repo,
older than the current content hash. Prevents local disk usage
explosion. Called both after bootstrapping from cache, and as part of
`bootstrap_docker.sh`.
* Reduced `retry` from 10s to 5s. Can be disabled with
`RETRY_DISABLED=1` env var.

avm-transpiler/Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:bookworm
+FROM rust:bullseye
 
 WORKDIR /usr/src
 COPY ./avm-transpiler ./avm-transpiler
@@ -8,6 +8,6 @@ WORKDIR /usr/src/avm-transpiler
 RUN apt-get update && apt-get install -y git
 RUN ./scripts/bootstrap_native.sh
 
-FROM ubuntu:lunar
+FROM ubuntu:focal
 COPY --from=0 /usr/src/avm-transpiler/target/release/avm-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler
-ENTRYPOINT ["sh", "-c"]
+ENTRYPOINT ["sh", "-c"]

avm-transpiler/bootstrap.sh

@@ -16,4 +16,7 @@ if [ -n "$CMD" ]; then
   fi
 fi
 
+# Attempt to just pull artefacts from CI and exit on success.
+./bootstrap_cache.sh && exit
+
 ./scripts/bootstrap_native.sh

avm-transpiler/bootstrap_cache.sh

@@ -1,9 +1,13 @@
 #!/usr/bin/env bash
 set -eu
 
+[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1
+
 cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m"
 extract_repo avm-transpiler \
   /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/
+
+remove_old_images avm-transpiler

barretenberg/cpp/bootstrap.sh

@@ -31,6 +31,9 @@ fi
 # Download ignition transcripts.
 (cd ./srs_db && ./download_ignition.sh 0)
 
+# Attempt to just pull artefacts from CI and exit on success.
+./bootstrap_cache.sh && exit
+
 # Pick native toolchain file.
 ARCH=$(uname -m)
 if [ "$OS" == "macos" ]; then

barretenberg/cpp/bootstrap_cache.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -eu
+
+[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1
+
+cd "$(dirname "$0")"
+source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m"
+extract_repo barretenberg-wasm-linux-clang \
+  /usr/src/barretenberg/cpp/build-wasm/bin ./cpp/build-wasm \
+  /usr/src/barretenberg/cpp/build-wasm-threads/bin ./cpp/build-wasm-threads
+
+remove_old_images barretenberg-wasm-linux-clang

barretenberg/ts/bootstrap.sh

@@ -18,10 +18,13 @@ if [ -n "$CMD" ]; then
   fi
 fi
 
+# Attempt to just pull artefacts from CI and exit on success.
+./bootstrap_cache.sh && exit
+
 yarn install --immutable
 echo "Building with command 'yarn $BUILD_CMD'..."
 yarn $BUILD_CMD
 
 # Make bin globally available.
 npm link
-echo "Barretenberg ts build successful"
+echo "Barretenberg ts build successful"

barretenberg/ts/bootstrap_cache.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -eu
+
+[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1
+
+cd "$(dirname "$0")"
+source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m"
+extract_repo bb.js /usr/src/barretenberg/ts/dest .
+# Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled.
+yarn install
+
+remove_old_images bb.js

bootstrap.sh

@@ -32,6 +32,8 @@ if [ -n "$CMD" ]; then
     git clean -ffdx
 
     exit 0
+  elif [ "$CMD" = "full" ]; then
+    export NO_CACHE=1
   else
     echo "Unknown command: $CMD"
     exit 1
@@ -61,19 +63,11 @@ PROJECTS=(
 
 # Build projects locally
 for P in "${PROJECTS[@]}"; do
-  if [ -n "${BOOTSTRAP_USE_REMOTE_CACHE:-}" ] && [ -f "$P/bootstrap_cache.sh" ]; then
-    echo "**************************************"
-    echo -e "\033[1mBootstrapping $P from remote cache...\033[0m"
-    echo "**************************************"
-    echo
-    $P/bootstrap_cache.sh
-  else
-    echo "**************************************"
-    echo -e "\033[1mBootstrapping $P...\033[0m"
-    echo "**************************************"
-    echo
-    $P/bootstrap.sh
-  fi
+  echo "**************************************"
+  echo -e "\033[1mBootstrapping $P...\033[0m"
+  echo "**************************************"
+  echo
+  (cd $P && ./bootstrap.sh)
   echo
   echo
 done

build-system/scripts/build_local

@@ -96,24 +96,27 @@ for E in "${PROJECTS[@]}"; do
 
   if (cd $(git rev-parse --show-toplevel) && git diff-index --name-only HEAD; git ls-files --others --exclude-standard) | grep -qE "$GREP_PATTERN"; then
     echo -e "${YELLOW}Project or dependency has local modifications! Building...${RESET}"
-    docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $DEPLOY_IMAGE_URI .
+    docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI .
   else
     if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w "$CACHE_IMAGE_URI$"; then
       echo -e "${GREEN}Image exists locally. Tagging as $DEPLOY_IMAGE_URI${RESET}"
-      docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI
     else
       if [ -z "$NO_CACHE" ] && [ -f ~/.aws/credentials ] && ecr_login && image_exists $REPO $TAG ; then
         docker pull $CACHE_IMAGE_URI
       else
         docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI .
       fi
-      docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI
     fi
   fi
 
+  # Retag with deployment uri.
+  docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI
+
   # Retag for aztecprotocol dockerhub.
   docker tag $DEPLOY_IMAGE_URI aztecprotocol/$REPO:latest
 
+  remove_old_images $REPO
+
   echo -e "${BOLD}Tagged${RESET}: aztecprotocol/$REPO:latest"
   echo -e "${BOLD}SHA256${RESET}: $(docker inspect --format='{{.Id}}' $DEPLOY_IMAGE_URI)"
 

build-system/scripts/create_ecr_manifest

@@ -24,7 +24,7 @@ IFS=','
 for A in $ARCH_LIST; do
   IMAGE_URI=$(calculate_image_uri $REPOSITORY $A)
   echo "Adding image $IMAGE_URI to manifest list $MULTIARCH_IMAGE_URI..."
-  docker manifest create $MULTIARCH_IMAGE_URI --amend $IMAGE_URI
+  retry docker manifest create $MULTIARCH_IMAGE_URI --amend $IMAGE_URI
 done
 
 retry docker manifest push --purge $MULTIARCH_IMAGE_URI

build-system/scripts/remove_old_images

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Removes all cache-* docker images for the given repository that are not the current content hash.
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+REPOSITORY=$1
+shift
+
+IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY)
+for IMAGE in $(docker images --format "{{.ID}}" $ECR_URL/$REPOSITORY --filter "before=$IMAGE_COMMIT_URI"); do
+  echo "Removing $IMAGE..."
+  docker rmi --force $IMAGE
+done

build-system/scripts/retry

@@ -1,7 +1,13 @@
+if [ -n "${RETRY_DISABLED:-}" ]; then
+    "$@" && exit || exit 1
+fi
+
 ATTEMPTS=3
-# Retries up to 3 times with 10 second intervals
+# Retries up to 3 times with 5 second intervals
 for i in $(seq 1 $ATTEMPTS); do
-    "$@" && exit || sleep 10
+    "$@" && exit
+    [ "$i" != "$ATTEMPTS" ] && sleep 5
 done
+
 >&2 echo "$@ failed after $ATTEMPTS attempts"
 exit 1

build-system/scripts/setup_env

@@ -6,6 +6,10 @@
 # The script should be sourced from the root of the repository, e.g:
 #   source ./build-system/scripts/setup_env
 # This ensures the resultant variables are set in the calling shell.
+
+# Save current options as we might be sourcing.
+CURRENT_SHELL_OPTS=$(set +o)
+trap 'eval "$CURRENT_SHELL_OPTS"' EXIT
 [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
@@ -139,6 +143,3 @@ cat $BASH_ENV
 
 # Having written the variables to $BASH_ENV, we now want to set them in this shell context.
 source $BASH_ENV
-
-# This script is often "sourced", we don't want calling shell to exit on a subsequent error!
-set +e

l1-contracts/Dockerfile

@@ -13,3 +13,6 @@ RUN forge clean && forge fmt --check && forge build && forge test
 RUN yarn && yarn lint
 RUN git add . && yarn slither && yarn slither-has-diff
 RUN forge build
+
+FROM scratch
+COPY --from=0 /usr/src/l1-contracts/out /usr/src/l1-contracts/out

l1-contracts/bootstrap.sh

@@ -15,12 +15,15 @@ if [ -n "$CMD" ]; then
   fi
 fi
 
-# Clean
-rm -rf broadcast cache out serve
-
 # Install foundry.
 . ./scripts/install_foundry.sh
 
+# Attempt to just pull artefacts from CI and exit on success.
+./bootstrap_cache.sh && exit
+
+# Clean
+rm -rf broadcast cache out serve
+
 # Install
 forge install --no-commit
 

l1-contracts/bootstrap_cache.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -eu
+
+[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1
+
+cd "$(dirname "$0")"
+source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+echo -e "\033[1mRetrieving contracts from remote cache...\033[0m"
+extract_repo l1-contracts /usr/src/l1-contracts/out .
+
+remove_old_images l1-contracts

noir-projects/Dockerfile

@@ -1,21 +1,21 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/avm-transpiler as transpiler
-FROM ubuntu:lunar
 
+FROM ubuntu:lunar AS builder
 # Copy in nargo
 COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo
 # Copy in transpiler
 COPY --from=transpiler /usr/src/avm-transpiler/target/release/avm-transpiler /usr/src/avm-transpiler/target/release/avm-transpiler
-
-WORKDIR /usr/src/noir-projects
 # Copy in noir projects
+WORKDIR /usr/src/noir-projects
 COPY . .
 # Build
 WORKDIR /usr/src/noir-projects/noir-contracts
 RUN ./scripts/compile.sh && ./scripts/transpile.sh && ../../noir/target/release/nargo test --silence-warnings
-
 WORKDIR /usr/src/noir-projects/noir-protocol-circuits
 RUN cd src && ../../../noir/target/release/nargo compile --silence-warnings && ../../../noir/target/release/nargo test --silence-warnings
-
 WORKDIR /usr/src/noir-projects/aztec-nr
 RUN ../../noir/target/release/nargo compile --silence-warnings && ../../noir/target/release/nargo test --silence-warnings
+
+FROM scratch
+COPY --from=builder /usr/src/noir-projects /usr/src/noir-projects

noir-projects/bootstrap.sh

@@ -3,6 +3,21 @@ set -eu
 
 cd "$(dirname "$0")"
 
+CMD=${1:-}
+
+if [ -n "$CMD" ]; then
+  if [ "$CMD" = "clean" ]; then
+    git clean -fdx
+    exit 0
+  else
+    echo "Unknown command: $CMD"
+    exit 1
+  fi
+fi
+
+# Attempt to just pull artefacts from CI and exit on success.
+./bootstrap_cache.sh && exit
+
 PROJECTS=(
   noir-contracts
   noir-protocol-circuits

noir-projects/bootstrap_cache.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -eu
+
+[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1
+
+cd "$(dirname "$0")"
+source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m"
+extract_repo noir-projects \
+  /usr/src/noir-projects/noir-contracts/target ./noir-contracts \
+  /usr/src/noir-projects/noir-protocol-circuits/src/target ./noir-protocol-circuits/src
+
+remove_old_images noir-projects

noir/Dockerfile

@@ -9,4 +9,4 @@ FROM ubuntu:focal
 # Install git as nargo needs it to clone.
 RUN apt-get update && apt-get install -y git tini && rm -rf /var/lib/apt/lists/* && apt-get clean
 COPY --from=0 /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo
-ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"]
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/noir/target/release/nargo"]

noir/bootstrap.sh

@@ -15,5 +15,8 @@ if [ -n "$CMD" ]; then
   fi
 fi
 
+# Attempt to just pull artefacts from CI and exit on success.
+./bootstrap_cache.sh && exit
+
 ./scripts/bootstrap_native.sh
 ./scripts/bootstrap_packages.sh

noir/bootstrap_cache.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -eu
 
+[ -z "${NO_CACHE:-}" ] && type docker &> /dev/null && [ -f ~/.aws/credentials ] || exit 1
+
 cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
@@ -9,3 +11,5 @@ extract_repo noir-packages /usr/src/noir/packages ./
 echo -e "\033[1mRetrieving nargo from remote cache...\033[0m"
 extract_repo noir /usr/src/noir/target/release ./target/
 
+remove_old_images noir-packages
+remove_old_images noir