Multi-platform caching for buildx

examples/aws-container-registry/ts/index.ts

@@ -51,6 +51,12 @@ const buildxImage = new docker.buildx.Image("buildx", {
   exports: ["type=registry"],
   file: "app/Dockerfile",
   platforms: ["linux/arm64", "linux/amd64"],
+  cacheTo: [
+    pulumi.interpolate`type=registry,mode=max,image-manifest=true,oci-mediatypes=true,ref=${repo.repositoryUrl}:cache`,
+  ],
+  cacheFrom: [
+    pulumi.interpolate`type=registry,ref=${repo.repositoryUrl}:cache`,
+  ],
   context: "app",
   registries: [
     {

examples/docker-container-registry/ts/index.ts

@@ -33,6 +33,8 @@ const buildxImage = new docker.buildx.Image("my-buildx-image", {
   tags: [`${imageName}:buildx`],
   exports: ["type=registry"],
   platforms: ["linux/arm64", "linux/amd64"],
+  cacheFrom: ["type=gha", `type=registry,ref=docker.io/${imageName}`],
+  cacheTo: ["type=gha", `type=registry,ref=docker.io/${imageName},mode=max`],
   context: "app",
   file: "app/Dockerfile",
   registries: [

provider/internal/image.go

@@ -2,11 +2,13 @@ package internal
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"reflect"
+	"strings"
 
 	// These imports are needed to register the drivers with buildkit.
 	_ "github.com/docker/buildx/driver/docker-container"
@@ -20,6 +22,7 @@ import (
 	"github.com/docker/docker/errdefs"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
 	"github.com/muesli/reflow/dedent"
+	"github.com/opencontainers/go-digest"
 
 	provider "github.com/pulumi/pulumi-go-provider"
 	"github.com/pulumi/pulumi-go-provider/infer"
@@ -208,6 +211,122 @@ func (ia *ImageArgs) buildable() bool {
 	return reflect.DeepEqual(ia, &filtered)
 }
 
+func (ia ImageArgs) toBuilds(
+	ctx provider.Context,
+	preview bool,
+) ([]controllerapi.BuildOptions, error) {
+	opts, err := ia.toBuildOptions(preview)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if we need a workaround for https://github.com/docker/buildx/issues/1044
+	if len(ia.CacheTo) == 0 || len(ia.Platforms) <= 1 {
+		return []controllerapi.BuildOptions{opts}, nil
+	}
+
+	// Split the build into N pieces: one build with only local caching, and an
+	// additional cache-only build for each platform.
+	builds := []controllerapi.BuildOptions{}
+
+	origCacheTo := opts.CacheTo
+
+	// Build 1:
+	// - No --cache-to.
+	// - Extend --cache-from with platform-specific caches, while preserving existing ones.
+	// - Preserve exports.
+	opts.CacheTo = nil
+	opts.CacheFrom = append(opts.CacheFrom, cachesFor(ctx, opts.CacheFrom, opts.Platforms...)...)
+	builds = append(builds, opts)
+
+	// Build 2..P for each platform:
+	// - --output=type=cacheonly.
+	// - No --cache-from (rely on local build cache).
+	// - --cache-to
+	for _, p := range opts.Platforms {
+		// Only build for this platform.
+		opts.Platforms = []string{p}
+		// Only export caches.
+		opts.Exports = []*controllerapi.ExportEntry{{Type: "cacheonly"}}
+		// Rely on local build cache.
+		opts.CacheFrom = nil
+		// Cache to platform-aware tags.
+		opts.CacheTo = cachesFor(ctx, origCacheTo, p)
+		builds = append(builds, opts)
+	}
+
+	return builds, nil
+}
+
+// cachesFor is a workaround for https://github.com/docker/buildx/issues/1044
+// which modifies the names of cache to/from entries to be platform-aware.
+func cachesFor(
+	ctx provider.Context,
+	existing []*controllerapi.CacheOptionsEntry,
+	platforms ...string,
+) []*controllerapi.CacheOptionsEntry {
+	caches := []*controllerapi.CacheOptionsEntry{}
+
+	// Iterate over existing cache entries first to preserve precedence.
+	for _, c := range existing {
+		for _, p := range platforms {
+			entry := &controllerapi.CacheOptionsEntry{
+				Type:  c.Type,
+				Attrs: make(map[string]string),
+			}
+			for k, v := range c.Attrs {
+				entry.Attrs[k] = v
+			}
+			plat := strings.Replace(p, "/", "-", -1)
+
+			switch c.Type {
+			case "gha":
+				entry.Attrs["scope"] += "-" + plat
+			case "s3", "azblob":
+				if entry.Attrs["name"] != "" {
+					entry.Attrs["name"] += "-" + plat
+				} else {
+					entry.Attrs["name"] = plat
+				}
+			case "registry":
+				// We don't want these synthetic caches to show up as tags on
+				// registries, so instead we identify them as opaque blobs
+				// whose names are derived from the base ref + platform.
+				h := sha256.New()
+				h.Write([]byte(entry.Attrs["ref"]))
+				h.Write([]byte(p))
+				dgst := digest.NewDigest(digest.SHA256, h)
+
+				ref, err := reference.Parse(entry.Attrs["ref"])
+				if err != nil {
+					ctx.Log(
+						diag.Warning,
+						fmt.Errorf("Unable to parse cache entry: %w", err).Error(),
+					)
+					continue
+				}
+
+				if named, ok := ref.(reference.Named); ok {
+					named = reference.TrimNamed(named)
+					ref, _ = reference.WithDigest(named, dgst) // Can't error.
+				} else {
+					named, err := reference.WithName(ref.String())
+					if err != nil {
+						ctx.Log(diag.Warning, fmt.Errorf("Unable to build cache key: %w", err).Error())
+						continue
+					}
+					ref, _ = reference.WithDigest(named, dgst) // Can't error.
+				}
+
+				entry.Attrs["ref"] = ref.String()
+			default:
+			}
+			caches = append(caches, entry)
+		}
+	}
+	return caches
+}
+
 func (ia *ImageArgs) toBuildOptions(preview bool) (controllerapi.BuildOptions, error) {
 	var multierr error
 
@@ -299,9 +418,9 @@ func (i *Image) Update(
 		return state, fmt.Errorf("buildkit is not supported on this host")
 	}
 
-	opts, err := input.toBuildOptions(preview)
+	builds, err := input.toBuilds(ctx, preview)
 	if err != nil {
-		return state, fmt.Errorf("validating input: %w", err)
+		return state, fmt.Errorf("preparing: %w", err)
 	}
 
 	hash, err := HashContext(input.Context, input.File)
@@ -318,20 +437,26 @@ func (i *Image) Update(
 		return state, nil
 	}
 
-	result, err := cfg.client.Build(ctx, opts)
-	if err != nil {
-		return state, fmt.Errorf("building %q: %w", input.Tags, err)
-	}
-
 	var id string
-	if digest, ok := result.ExporterResponse["containerimage.digest"]; ok {
-		id = digest
-	} else if digest, ok := result.ExporterResponse[exptypes.ExporterImageConfigDigestKey]; ok {
-		id = digest
-	} else if tags, ok := result.ExporterResponse["image.name"]; ok {
-		id = tags
-	} else {
-		id = name
+
+	for _, b := range builds {
+		result, err := cfg.client.Build(ctx, b)
+		if err != nil {
+			return state, fmt.Errorf("building %q: %w", input.Tags, err)
+		}
+		if id != "" {
+			continue
+		}
+
+		if digest, ok := result.ExporterResponse["containerimage.digest"]; ok {
+			id = digest
+		} else if digest, ok := result.ExporterResponse[exptypes.ExporterImageConfigDigestKey]; ok {
+			id = digest
+		} else if tags, ok := result.ExporterResponse["image.name"]; ok {
+			id = tags
+		} else {
+			id = name
+		}
 	}
 
 	// TODO: Handle case with no export.
@@ -389,6 +514,7 @@ func (*Image) Read(
 				// No manifest to read if we didn't push.
 				continue
 			}
+
 			for _, tag := range input.Tags {
 				expectManifest = true
 				infos, err := cfg.client.Inspect(ctx, tag)

provider/internal/image_test.go

@@ -656,3 +656,37 @@ func TestBuildable(t *testing.T) {
 		})
 	}
 }
+
+func TestToBuilds(t *testing.T) {
+	t.Run("multi-platform caching", func(t *testing.T) {
+		ia := ImageArgs{
+			Tags:      []string{"foo"},
+			Platforms: []string{"linux/amd64", "linux/arm64"},
+			CacheTo:   []string{"type=gha,mode=max", "type=registry,ref=foo"},
+			CacheFrom: []string{"type=s3,name=bar", "type=registry,ref=foo"},
+		}
+
+		builds, err := ia.toBuilds(nil, false)
+		assert.NoError(t, err)
+
+		assert.Len(t, builds, 3)
+
+		// Build 1
+		assert.Nil(t, builds[0].CacheTo)
+		assert.Len(t, builds[0].CacheFrom, len(ia.CacheFrom)*(1+len(ia.Platforms)))
+
+		// Build 2
+		assert.Nil(t, builds[1].CacheFrom)
+		assert.Len(t, builds[2].Platforms, 1)
+		assert.Equal(t, "linux/amd64", builds[1].Platforms[0])
+		assert.Len(t, builds[1].Exports, 1)
+		assert.Equal(t, "cacheonly", builds[1].Exports[0].Type)
+
+		// Build 3
+		assert.Nil(t, builds[2].CacheFrom)
+		assert.Len(t, builds[2].Platforms, 1)
+		assert.Equal(t, "linux/arm64", builds[2].Platforms[0])
+		assert.Len(t, builds[2].Exports, 1)
+		assert.Equal(t, "cacheonly", builds[2].Exports[0].Type)
+	})
+}