Skip to content

Commit

Permalink
Fix Docker Engine RPC buffer overflows. (#843)
Browse files Browse the repository at this point in the history 
Fix Docker Engine RPC buffer overflows.

Fixes #812, #778, #573.

In issue [#573](#573),
users would sometimes report the image push fails with an error "the
image does not exist locally with the tag". This is likely due to Docker
Engine RPCs overflowing the default `bufio.Scanner` maximum buffer size
of 64k. Replacing the scanner with a `json.Decoder` enables us to handle
large responses. When the image build fails, users should now see a
warning or error before the provider pushes the built image.

In issue [#812](#812),
users would see the update operation on the Image resource succeed, but
the output `repoDigest` would not change. In this PR, if for any reason
the image fails to build, or update the Docker Engine's image store,
push, or we detect inconsistency between the image ID returned by the
`ImageBuild` RPC and what we expected, we fail the resource operation
with an error.

In issue [#778](#778), a
repo digest would be chosen that doesn't match the name (sans tag) of
the image pushed. As part of the additional checks, we fix this and
ensure the normalized name of the repo digest matches the input image
name.

Additionally, in case there is a data race between `ImageBuild` and
`ImagePush`, we extract the expected image IDs and repo digests from
those operations and compare to the image store. As this is a net-new
check and we could not reproduce a repo digest mismatch, this is a
warning.

In a follow-up (#846) we may consider removing some of the defensive
programming in this change.
  • Loading branch information
@AaronFriel
AaronFriel authored Nov 16, 2023
1 parent a14dab6 commit ce362cb
Show file tree
Hide file tree
Showing 32 changed files with 505 additions and 168 deletions.
1 change: 1 addition & 0 deletions examples/aws-container-registry/csharp/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ static Task<int> Main() => Deployment.RunAsync(async () => {
{
{ "baseImageName", image.BaseImageName },
{ "fullImageName", image.ImageName },
{ "repoDigest", image.RepoDigest },
};
});
}
1 change: 1 addition & 0 deletions examples/aws-container-registry/go/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func main() {

// Export the resulting image name
ctx.Export("fullImageName", image.ImageName)
ctx.Export("repoDigest", image.RepoDigest)
return nil
})
}
1 change: 1 addition & 0 deletions examples/aws-container-registry/py/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,4 @@ def getRegistryInfo(rid):
# Export the resulting base name in addition to the specific version pushed.
pulumi.export('baseImageName', image.base_image_name)
pulumi.export('imageName', image.image_name)
pulumi.export('repoDigest', image.repo_digest)
1 change: 1 addition & 0 deletions examples/azure-container-registry/csharp/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ static Task<int> Main() => Deployment.RunAsync(async () => {
{
{ "baseImageName", image.BaseImageName },
{ "fullImageName", image.ImageName },
{ "repoDigest", image.RepoDigest },
};
});
}
1 change: 1 addition & 0 deletions examples/azure-container-registry/go/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ func main() {
}

ctx.Export("deps-image", image.ImageName)
ctx.Export("repoDigest", image.RepoDigest)

return nil
})
Expand Down
1 change: 1 addition & 0 deletions examples/azure-container-registry/py/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,4 @@

# Export the resulting image name
pulumi.export('imageName', image.image_name)
pulumi.export('repoDigest', image.repo_digest)
1 change: 1 addition & 0 deletions examples/azure-container-registry/ts/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,4 @@ const image = new docker.Image("my-image", {

// Export the resulting image name
export const fullImageName = image.imageName;
export const repoDigest = image.repoDigest;
1 change: 1 addition & 0 deletions examples/azure-container-registry/ts/step2/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,4 @@ const image = new docker.Image("my-image", {

// Export the resulting image name
export const fullImageName = image.imageName;
export const repoDigest = image.repoDigest;
1 change: 1 addition & 0 deletions examples/digitalocean-container-registry/csharp/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ static Task<int> Main() => Deployment.RunAsync(() => {
{
{ "baseImageName", image.BaseImageName },
{ "fullImageName", image.ImageName },
{ "repoDigest", image.RepoDigest },
};
});
}
1 change: 1 addition & 0 deletions examples/digitalocean-container-registry/go/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func main() {

// Export the resulting image name
ctx.Export("fullImageName", image.ImageName)
ctx.Export("repoDigest", image.RepoDigest)
return nil
})
}
1 change: 1 addition & 0 deletions examples/digitalocean-container-registry/py/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,4 @@ def getRegistryInfo(info):
# Export the resulting base name in addition to the specific version pushed.
pulumi.export('baseImageName', image.base_image_name)
pulumi.export('fullImageName', image.image_name)
pulumi.export('repoDigest', image.repo_digest)
1 change: 1 addition & 0 deletions examples/digitalocean-container-registry/ts/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,4 @@ const image = new docker.Image("my-image", {
// Export the resuling base name in addition to the specific version pushed.
export const baseImageName = image.baseImageName;
export const fullImageName = image.imageName;
export const repoDigest = image.repoDigest;
47 changes: 33 additions & 14 deletions examples/docker-container-registry/go/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,64 +3,83 @@ module cbp-aws-go
go 1.21

require (
github.com/pulumi/pulumi-docker/sdk/v4 v4.0.0-alpha.4
github.com/pulumi/pulumi/sdk/v3 v3.55.0
github.com/pulumi/pulumi-docker/sdk/v4 v4.4.5
github.com/pulumi/pulumi/sdk/v3 v3.91.1
)

require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/charmbracelet/bubbles v0.16.1 // indirect
github.com/charmbracelet/bubbletea v0.24.2 // indirect
github.com/charmbracelet/lipgloss v0.7.1 // indirect
github.com/cheggaaa/pb v1.0.29 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
github.com/djherbis/times v1.5.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/go-git/go-git/v5 v5.5.1 // indirect
github.com/gofrs/uuid v4.2.0+incompatible // indirect
github.com/go-git/go-billy/v5 v5.4.0 // indirect
github.com/go-git/go-git/v5 v5.6.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.1.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/hcl/v2 v2.17.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.15.1 // indirect
github.com/opentracing/basictracer-go v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pjbgf/sha1cd v0.2.3 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/term v1.1.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
github.com/pulumi/esc v0.5.6 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/skeema/knownhosts v1.1.0 // indirect
github.com/spf13/cast v1.4.1 // indirect
github.com/spf13/cobra v1.6.1 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.2.0 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/zclconf/go-cty v1.13.2 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230725213213-b022f6e96895 // indirect
google.golang.org/grpc v1.56.3 // indirect
google.golang.org/grpc v1.57.1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
Expand Down
Loading

0 comments on commit ce362cb

Please sign in to comment.