From b2342f587442f9f56f61a21105dffb8549b316ad Mon Sep 17 00:00:00 2001 From: Nikhil Benesch Date: Mon, 6 Dec 2021 13:22:04 -0500 Subject: [PATCH 1/3] Revert "Use the docker CLI's build context construction API" This reverts commit 32c8216359aaea67238c969b94f266d351e35ed2. Docker context hashing is not stable. --- cmd/pulumi-resource-docker-buildkit/main.go | 72 ++++++++++++++++----- go.mod | 4 +- go.sum | 20 ------ 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/cmd/pulumi-resource-docker-buildkit/main.go b/cmd/pulumi-resource-docker-buildkit/main.go index 795d97b..4a779e0 100644 --- a/cmd/pulumi-resource-docker-buildkit/main.go +++ b/cmd/pulumi-resource-docker-buildkit/main.go @@ -15,6 +15,7 @@ package main import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -26,9 +27,10 @@ import ( "strings" "sync" - "github.com/docker/cli/cli/command/image/build" + "github.com/docker/docker/pkg/fileutils" pbempty "github.com/golang/protobuf/ptypes/empty" structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/moby/buildkit/frontend/dockerfile/dockerignore" "github.com/pulumi/pulumi/pkg/v3/resource/provider" "github.com/pulumi/pulumi/sdk/v3/go/common/diag" "github.com/pulumi/pulumi/sdk/v3/go/common/resource" @@ -107,7 +109,7 @@ func (k *dockerBuildkitProvider) Diff(ctx context.Context, req *rpc.DiffRequest) applyDefaults(news) news["registryServer"] = news["registry"].ObjectValue()["server"] delete(news, "registry") - contextDigest, err := hashContext(news["context"].StringValue(), news["dockerfile"].StringValue()) + contextDigest, err := hashContext(news["context"].StringValue()) if err != nil { return nil, err } @@ -226,7 +228,7 @@ func (k *dockerBuildkitProvider) dockerBuild( username := registry["username"] password := registry["password"] - contextDigest, err := hashContext(context, dockerfile) + contextDigest, err := hashContext(context) if err != nil { return nil, err } @@ -338,23 +340,63 @@ func (w *logWriter) Write(p []byte) (n int, err error) { return len(p), w.host.Log(w.ctx, w.severity, w.urn, string(p)) } -func hashContext(contextPath, dockerfile string) (string, error) { - dr, err := os.Open(filepath.Join(contextPath, dockerfile)) +func hashContext(contextPath string) (string, error) { + dockerIgnore, err := os.ReadFile(filepath.Join(contextPath, ".dockerignore")) + if err != nil && !os.IsNotExist(err) { + return "", fmt.Errorf("unable to read .dockerignore file: %w", err) + } + ignorePatterns, err := dockerignore.ReadAll(bytes.NewReader(dockerIgnore)) if err != nil { - return "", fmt.Errorf("could not find %q: %w", dockerfile, err) + return "", fmt.Errorf("unable to parse .dockerignore file: %w", err) } - defer dr.Close() - - h := sha256.New() - buildCtx, _, err := build.GetContextFromReader(dr, dockerfile) + ignoreMatcher, err := fileutils.NewPatternMatcher(ignorePatterns) if err != nil { - return "", fmt.Errorf("constructing buildcontext: %w", err) + return "", fmt.Errorf("unable to load rules from .dockerignore: %w", err) } - defer buildCtx.Close() - - _, err = io.Copy(h, buildCtx) + var hashInput []byte + err = filepath.WalkDir(contextPath, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + path, err = filepath.Rel(contextPath, path) + if err != nil { + return err + } + if path == "." { + return nil + } + ignore, err := ignoreMatcher.Matches(path) + if err != nil { + return fmt.Errorf(".dockerignore rule failed: %w", err) + } + if ignore { + if d.IsDir() { + return filepath.SkipDir + } else { + return nil + } + } else if d.IsDir() { + return nil + } + f, err := os.Open(filepath.Join(contextPath, path)) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + h := sha256.New() + _, err = io.Copy(h, f) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + hashInput = append(hashInput, path...) + hashInput = append(hashInput, h.Sum(nil)...) + hashInput = append(hashInput, byte(0)) + return nil + }) if err != nil { - return "", fmt.Errorf("reading buildcontext: %w", err) + return "", fmt.Errorf("unable to hash build context: %w", err) } + h := sha256.New() + h.Write(hashInput) return hex.EncodeToString(h.Sum(nil)), nil } diff --git a/go.mod b/go.mod index 5e2bc79..83f3d37 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/MaterializeInc/pulumi-docker-buildkit go 1.16 require ( - github.com/docker/cli v20.10.7+incompatible + github.com/docker/docker v20.10.7+incompatible github.com/golang/protobuf v1.5.2 - github.com/moby/buildkit v0.9.0 // indirect + github.com/moby/buildkit v0.9.0 github.com/pkg/errors v0.9.1 github.com/pulumi/pulumi/pkg/v3 v3.0.0 github.com/pulumi/pulumi/sdk/v3 v3.0.0 diff --git a/go.sum b/go.sum index 54f466b..a33b0e5 100644 --- a/go.sum +++ b/go.sum @@ -81,7 +81,6 @@ github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxw github.com/Azure/azure-storage-blob-go v0.13.0/go.mod h1:pA9kNqtjUeQF2zOSu4s//nUdBD+e64lEuc4sVnuOfNs= github.com/Azure/go-amqp v0.13.0/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs= github.com/Azure/go-amqp v0.13.1/go.mod h1:qj+o8xPCz9tMSbQ83Vp8boHahuRDl5mkNHyt1xlxUTs= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -148,7 +147,6 @@ github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= @@ -158,7 +156,6 @@ github.com/Microsoft/hcsshim v0.8.10/go.mod h1:g5uw8EV2mAlzqe94tfNBNdr89fnbD/n3H github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.18 h1:cYnKADiM1869gvBpos3YCteeT6sZLB48lB5dmMMs8Tg= github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim/test v0.0.0-20200826032352-301c83a30e7c/go.mod h1:30A5igQ91GEmhYJF8TaRP79pMBOYynRsyOByfVV0dU4= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= @@ -284,7 +281,6 @@ github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= @@ -308,7 +304,6 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= -github.com/containerd/containerd v1.5.3 h1:mfKOepNDIJ3EiBTEyHFpEqB6YSOSkGcjPDIu7cD+YzY= github.com/containerd/containerd v1.5.3/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -316,7 +311,6 @@ github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= @@ -388,7 +382,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= @@ -414,7 +407,6 @@ github.com/docker/cli v0.0.0-20190925022749-754388324470/go.mod h1:JLrzqnKDaYBop github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v20.10.7+incompatible h1:pv/3NqibQKphWZiAskMzdz8w0PRbtTaEB+f6NwdU7Is= github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.6.0-rc.1.0.20180327202408-83389a148052+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -438,7 +430,6 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6Uezg github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libnetwork v0.8.0-dev.2.0.20200917202933-d0951081b35f/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= @@ -573,7 +564,6 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -928,18 +918,14 @@ github.com/moby/buildkit v0.9.0/go.mod h1:S9ceObCS/yMHsJD7FQx4fUCe3E7HHYjYVvk0Ct github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= github.com/moby/sys/mount v0.1.1/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74= -github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM= github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= github.com/moby/sys/mountinfo v0.1.0/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/symlink v0.1.0 h1:MTFZ74KtNI6qQQpuBxU+uKCim4WtOMokr03hCfJcazE= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2/go.mod h1:TjQg8pa4iejrUrjiz0MCtMV38jdMNW4doKSiBrEvCQQ= -github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf h1:Un6PNx5oMK6CCwO3QTUyPiK2mtZnPrpDl5UnZ64eCkw= github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -948,7 +934,6 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= @@ -993,10 +978,8 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -1004,7 +987,6 @@ github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2r github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE= -github.com/opencontainers/runc v1.0.0-rc93 h1:x2UMpOOVf3kQ8arv/EsDGwim8PTNqzL1/EYDr/+scOM= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1288,7 +1270,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/contrib v0.21.0/go.mod h1:EH4yDYeNoaTqn/8yCWQmfNB78VHfGX2Jt2bvnvzBlGM= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.21.0/go.mod h1:Vm5u/mtkj1OMhtao0v+BGo2LUoLCgHYXvRmj0jWITlE= @@ -1576,7 +1557,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 93fac8416c7669f52cd7a33c1eb31793679997bc Mon Sep 17 00:00:00 2001 From: Nikhil Benesch Date: Mon, 6 Dec 2021 13:27:49 -0500 Subject: [PATCH 2/3] Always add the Dockerfile to the context hash The Dockerfile might not be inside the context, but it should still go in the context hash, since we need to trigger a rebuild if it changes. Fix #5. --- cmd/pulumi-resource-docker-buildkit/main.go | 27 ++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/cmd/pulumi-resource-docker-buildkit/main.go b/cmd/pulumi-resource-docker-buildkit/main.go index 4a779e0..ff761d8 100644 --- a/cmd/pulumi-resource-docker-buildkit/main.go +++ b/cmd/pulumi-resource-docker-buildkit/main.go @@ -109,7 +109,10 @@ func (k *dockerBuildkitProvider) Diff(ctx context.Context, req *rpc.DiffRequest) applyDefaults(news) news["registryServer"] = news["registry"].ObjectValue()["server"] delete(news, "registry") - contextDigest, err := hashContext(news["context"].StringValue()) + contextDigest, err := hashContext( + news["context"].StringValue(), + news["dockerfile"].StringValue(), + ) if err != nil { return nil, err } @@ -228,7 +231,7 @@ func (k *dockerBuildkitProvider) dockerBuild( username := registry["username"] password := registry["password"] - contextDigest, err := hashContext(context) + contextDigest, err := hashContext(context, dockerfile) if err != nil { return nil, err } @@ -340,7 +343,7 @@ func (w *logWriter) Write(p []byte) (n int, err error) { return len(p), w.host.Log(w.ctx, w.severity, w.urn, string(p)) } -func hashContext(contextPath string) (string, error) { +func hashContext(contextPath string, dockerfile string) (string, error) { dockerIgnore, err := os.ReadFile(filepath.Join(contextPath, ".dockerignore")) if err != nil && !os.IsNotExist(err) { return "", fmt.Errorf("unable to read .dockerignore file: %w", err) @@ -396,6 +399,24 @@ func hashContext(contextPath string) (string, error) { if err != nil { return "", fmt.Errorf("unable to hash build context: %w", err) } + // Add the Dockerfile hash directly, since it might not be in the build + // context. + { + path := filepath.Join(contextPath, dockerfile) + f, err := os.Open(path) + if err != nil { + return "", fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + h := sha256.New() + _, err = io.Copy(h, f) + if err != nil { + return "", fmt.Errorf("read %s: %w", path, err) + } + hashInput = append(hashInput, path...) + hashInput = append(hashInput, h.Sum(nil)...) + hashInput = append(hashInput, byte(0)) + } h := sha256.New() h.Write(hashInput) return hex.EncodeToString(h.Sum(nil)), nil From e792ea32a704138526c69256ffb174a11ef066fb Mon Sep 17 00:00:00 2001 From: Andreas Fuchs Date: Tue, 7 Dec 2021 12:19:28 +0100 Subject: [PATCH 3/3] Refactor single-file hashing & account for file modes, too This requires less copy&pasted code and lets us invalidate the build context if a file mode changed, as well (this is rare, but might happen). --- cmd/pulumi-resource-docker-buildkit/main.go | 75 ++++++++++++--------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/cmd/pulumi-resource-docker-buildkit/main.go b/cmd/pulumi-resource-docker-buildkit/main.go index ff761d8..4317672 100644 --- a/cmd/pulumi-resource-docker-buildkit/main.go +++ b/cmd/pulumi-resource-docker-buildkit/main.go @@ -21,6 +21,7 @@ import ( "encoding/hex" "fmt" "io" + "io/fs" "os" "os/exec" "path/filepath" @@ -343,6 +344,39 @@ func (w *logWriter) Write(p []byte) (n int, err error) { return len(p), w.host.Log(w.ctx, w.severity, w.urn, string(p)) } +type contextHash struct { + contextPath string + input bytes.Buffer +} + +func newContextHash(contextPath string) *contextHash { + return &contextHash{contextPath: contextPath} +} + +func (ch *contextHash) hashPath(path string, fileMode fs.FileMode) error { + f, err := os.Open(filepath.Join(ch.contextPath, path)) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + h := sha256.New() + _, err = io.Copy(h, f) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + ch.input.Write([]byte(path)) + ch.input.Write([]byte(fileMode.String())) + ch.input.Write(h.Sum(nil)) + ch.input.WriteByte(0) + return nil +} + +func (ch *contextHash) hexSum() string { + h := sha256.New() + ch.input.WriteTo(h) + return hex.EncodeToString(h.Sum(nil)) +} + func hashContext(contextPath string, dockerfile string) (string, error) { dockerIgnore, err := os.ReadFile(filepath.Join(contextPath, ".dockerignore")) if err != nil && !os.IsNotExist(err) { @@ -356,7 +390,11 @@ func hashContext(contextPath string, dockerfile string) (string, error) { if err != nil { return "", fmt.Errorf("unable to load rules from .dockerignore: %w", err) } - var hashInput []byte + ch := newContextHash(contextPath) + err = ch.hashPath(dockerfile, 0) + if err != nil { + return "", fmt.Errorf("hashing dockerfile %q: %w", dockerfile, err) + } err = filepath.WalkDir(contextPath, func(path string, d os.DirEntry, err error) error { if err != nil { return err @@ -381,43 +419,18 @@ func hashContext(contextPath string, dockerfile string) (string, error) { } else if d.IsDir() { return nil } - f, err := os.Open(filepath.Join(contextPath, path)) + info, err := d.Info() if err != nil { - return fmt.Errorf("open %s: %w", path, err) + return fmt.Errorf("determining mode for %q: %w", path, err) } - defer f.Close() - h := sha256.New() - _, err = io.Copy(h, f) + err = ch.hashPath(path, info.Mode()) if err != nil { - return fmt.Errorf("read %s: %w", path, err) + return fmt.Errorf("hashing %q: %w", path, err) } - hashInput = append(hashInput, path...) - hashInput = append(hashInput, h.Sum(nil)...) - hashInput = append(hashInput, byte(0)) return nil }) if err != nil { return "", fmt.Errorf("unable to hash build context: %w", err) } - // Add the Dockerfile hash directly, since it might not be in the build - // context. - { - path := filepath.Join(contextPath, dockerfile) - f, err := os.Open(path) - if err != nil { - return "", fmt.Errorf("open %s: %w", path, err) - } - defer f.Close() - h := sha256.New() - _, err = io.Copy(h, f) - if err != nil { - return "", fmt.Errorf("read %s: %w", path, err) - } - hashInput = append(hashInput, path...) - hashInput = append(hashInput, h.Sum(nil)...) - hashInput = append(hashInput, byte(0)) - } - h := sha256.New() - h.Write(hashInput) - return hex.EncodeToString(h.Sum(nil)), nil + return ch.hexSum(), nil }