test: allow testing frontends from different version

This allows running Dockerfile tests so that the Dockerfile version and the BuildKit version are from different commits so that we can test that old Dockerfile releases remain compatible with the latest BuildKit. The tests are based on the commit of the Dockerfile frontend as we can't expect that new test would work on old frontends. In future we might consider doing it the other way as well but then we need a way to mark tests that can be ignored if they are not expected to pass because of a new feature dependency. Signed-off-by: Tonis Tiigi <[email protected]>
moby · Dec 13, 2024 · 1f94cab · 1f94cab
1 parent f209225
commit 1f94cab
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 27 deletions.
diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml
@@ -104,7 +104,6 @@ jobs:
       TESTFLAGS: "-v --parallel=6 --timeout=30m"
       GOTESTSUM_FORMAT: "standard-verbose"
       TEST_IMAGE_BUILD: "0"
-      TEST_IMAGE_ID: "buildkit-tests"
     strategy:
       fail-fast: false
       matrix:
@@ -161,7 +160,6 @@ jobs:
           targets: integration-tests
           set: |
             *.cache-from=type=gha,scope=${{ inputs.cache_scope }}
-            *.output=type=docker,name=${{ env.TEST_IMAGE_ID }}
         env:
           BUILDKITD_TAGS: ${{ matrix.tags }}
           TEST_COVERAGE: 1

diff --git a/docker-bake.hcl b/docker-bake.hcl
@@ -51,6 +51,24 @@ variable "TEST_COVERAGE" {
   default = null
 }
 
+variable "TEST_IMAGE_NAME" {
+  default = "buildkit-tests"
+}
+
+variable "TEST_CONTEXT" {
+  default = "."
+  description = "Context for building the test image"
+}
+
+variable "TEST_BINARIES_CONTEXT" {
+  default = TEST_CONTEXT
+  description = "Context for building the buildkitd for test image"
+}
+
+variable "BUILDKIT_SYNTAX" {
+  default = null
+}
+
 function "bindir" {
   params = [defaultdir]
   result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}"
@@ -117,12 +135,26 @@ target "integration-tests-base" {
   output = ["type=cacheonly"]
 }
 
+target "integration-tests-binaries" {
+  inherits = ["_common"]
+  target = "binaries"
+  context = TEST_BINARIES_CONTEXT
+}
+
 target "integration-tests" {
   inherits = ["integration-tests-base"]
   target = "integration-tests"
+  context = TEST_CONTEXT
+  contexts = TEST_CONTEXT != TEST_BINARIES_CONTEXT ? {
+    "binaries": "target:integration-tests-binaries"
+  } : null
   args = {
     GOBUILDFLAGS = TEST_COVERAGE == "1" ? "-cover" : null
+    BUILDKIT_SYNTAX = BUILDKIT_SYNTAX
   }
+  output = [
+    "type=docker,name=${TEST_IMAGE_NAME}",
+  ]
 }
 
 group "validate" {

diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release
@@ -104,6 +104,7 @@ case $TYP in
   buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \
     --build-arg "CHANNEL=$TAG" \
     --build-arg "BUILDTAGS=$buildTags" \
+    --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
     --output "${outputFlag},name=$REPO:$pushTag" \
     --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
     $currentcontext
@@ -125,6 +126,7 @@ case $TYP in
   buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \
     --build-arg "CHANNEL=$TAG" \
     --build-arg "BUILDTAGS=$buildTags" \
+    --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
     --output "$outputFlag" \
     --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
     $currentcontext

diff --git a/hack/images b/hack/images
@@ -112,5 +112,5 @@ if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then
   nocacheFilterFlag="--no-cache-filter=buildkit-export,gobuild-base,rootless"
 fi
 
-buildxCmd build --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \
+buildxCmd build --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" --build-arg BUILDKIT_DEBUG $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \
   $currentcontext
diff --git a/hack/test b/hack/test
@@ -15,14 +15,15 @@ set -eu -o pipefail
 
 : "${TEST_COVERAGE=}"
 : "${TEST_IMAGE_BUILD=1}"
-: "${TEST_IMAGE_ID=buildkit-tests}"
+: "${TEST_IMAGE_NAME=buildkit-tests}"
 : "${TEST_INTEGRATION=}"
 : "${TEST_GATEWAY=}"
 : "${TEST_DOCKERFILE=}"
 : "${TEST_DOCKERD=}"
 : "${TEST_DOCKERD_BINARY=$(which dockerd)}"
 : "${TEST_REPORT_SUFFIX=}"
 : "${TEST_KEEP_CACHE=}"
+: "${TEST_SUITE_CONTEXT=}"
 : "${TESTFLAGS=}"
 
 : "${DOCKERFILE_RELEASES=}"
@@ -87,20 +88,15 @@ if [ "$TEST_COVERAGE" == "1" ]; then
   export GO_TEST_COVERPROFILE="/testreports/coverage-report$TEST_REPORT_SUFFIX.txt"
 fi
 
+if [ -n "$TEST_SUITE_CONTEXT" ]; then
+  export TEST_BINARIES_CONTEXT=$currentcontext
+  # FIXME: something breaks with the syntax when using the context
+  export BUILDKIT_SYNTAX="docker/dockerfile:1.10.0"
+  currentcontext=$TEST_SUITE_CONTEXT
+fi
+
 if [ "$TEST_IMAGE_BUILD" = "1" ]; then
-  buildxCmd build $cacheFromFlags \
-    --build-arg ALPINE_VERSION \
-    --build-arg GO_VERSION \
-    --build-arg BUILDKITD_TAGS \
-    --build-arg HTTP_PROXY \
-    --build-arg HTTPS_PROXY \
-    --build-arg NO_PROXY \
-    --build-arg GOBUILDFLAGS \
-    --build-arg VERIFYFLAGS \
-    --build-arg CGO_ENABLED \
-    --target "integration-tests" \
-    --output "type=docker,name=$TEST_IMAGE_ID" \
-    $currentcontext
+  TEST_CONTEXT=$currentcontext buildxCmd bake integration-tests
 fi
 
 cacheVolume="buildkit-test-cache"
@@ -131,7 +127,7 @@ if [ "$TEST_INTEGRATION" == 1 ]; then
   cid=$(dockerCmd create $baseCreateFlags \
     ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} \
     -e SKIP_INTEGRATION_TESTS \
-    $TEST_IMAGE_ID \
+    $TEST_IMAGE_NAME \
     gotestsumandcover $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v})
   if [ "$TEST_DOCKERD" = "1" ]; then
     dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd
@@ -141,8 +137,8 @@ fi
 
 if [ "$TEST_GATEWAY" == 1 ]; then
   # Build-test "github.com/moby/buildkit/frontend/gateway/client", which isn't otherwise built by CI
-  # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_ID, which is a direct child of buildkit-base.
-  cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_ID go build -v ./frontend/gateway/client)
+  # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_NAME, which is a direct child of buildkit-base.
+  cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_NAME go build -v ./frontend/gateway/client)
   dockerCmd start -a $cid
 fi
 
@@ -165,7 +161,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
     tarout=$(mktemp -t dockerfile-frontend.XXXXXXXXXX)
 
     buildxCmd build $cacheFromFlags \
-      --build-arg "BUILDTAGS=$buildtags" \
+      --build-arg "BUILDTAGS=$buildtags" --build-arg "BUILDKIT_CONTEXT_KEEP_GIT_DIR=1" \
       --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \
       --output "type=oci,dest=$tarout" \
       $currentcontext
@@ -176,7 +172,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
           -e BUILDKIT_WORKER_RANDOM \
           -e FRONTEND_GATEWAY_ONLY=local:/$release.tar \
           -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend \
-          $TEST_IMAGE_ID \
+          $TEST_IMAGE_NAME \
           gotestsumandcover $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v})
         dockerCmd cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then

diff --git a/hack/util b/hack/util
@@ -36,12 +36,11 @@ buildAttestFlags() {
   fi
 }
 
-currentref=""
 currentcontext="."
 cacheFromFlags=""
 cacheToFlags=""
 if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ]; then
-  currentref="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF"
+  currentcontext="https://github.com/$GITHUB_REPOSITORY.git#$GITHUB_REF"
   if [ -n "$CACHE_FROM" ]; then
     for cfrom in $CACHE_FROM; do
       if [[ $cfrom == *"type=gha"* ]]; then
@@ -69,9 +68,6 @@ if [ "$GITHUB_ACTIONS" = "true" ] && [ "$GITHUB_REPOSITORY" = "moby/buildkit" ];
     done
   fi
 fi
-if [ -n "$currentref" ]; then
-  currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref"
-fi
 if [ -n "$CONTEXT" ]; then
   currentcontext=$CONTEXT
 fi