Skip to content

Commit

Permalink
delete remote tags
Browse files Browse the repository at this point in the history
  • Loading branch information
blampe committed Mar 15, 2024
1 parent 885d7cc commit 9743352
Show file tree
Hide file tree
Showing 32 changed files with 350 additions and 156 deletions.
36 changes: 0 additions & 36 deletions examples/aws-container-registry/ts/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import * as aws from "@pulumi/aws";
import * as docker from "@pulumi/docker";
import * as pulumi from "@pulumi/pulumi";
import * as random from "@pulumi/random";

// Create a private ECR registry.
Expand Down Expand Up @@ -43,38 +42,3 @@ const image = new docker.Image("my-image", {
// Export the resulting image name
export const imageName = image.baseImageName;
export const repoDigest = image.repoDigest;

// buildx

const buildxImage = new docker.buildx.Image("buildx", {
tags: [pulumi.interpolate`${repo.repositoryUrl}:buildx`],
platforms: ["linux/arm64", "linux/amd64"],
push: true,
cacheTo: [
{
registry: {
mode: "max",
imageManifest: true,
ociMediaTypes: true,
ref: pulumi.interpolate`${repo.repositoryUrl}:cache`,
},
},
],
cacheFrom: [
{
registry: {
ref: pulumi.interpolate`${repo.repositoryUrl}:cache`,
},
},
],
context: {
location: "app",
},
registries: [
{
address: registryInfo.server,
username: registryInfo.username,
password: registryInfo.password,
},
],
});
26 changes: 26 additions & 0 deletions examples/examples_nodejs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecr"
"github.com/pulumi/pulumi/pkg/v3/testing/integration"
"github.com/regclient/regclient/types/ref"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestBuildxTs(t *testing.T) {
Expand Down Expand Up @@ -108,6 +110,30 @@ func TestAwsContainerRegistryNode(t *testing.T) {
integration.ProgramTest(t, &test)
}

func TestAwsContainerRegistryBuildxNode(t *testing.T) {
region := os.Getenv("AWS_REGION")
if region == "" {
t.Skipf("Skipping test due to missing AWS_REGION environment variable")
}
test := getJsOptions(t).
With(integration.ProgramTestOptions{
Dir: path.Join(getCwd(t), "test-buildx/ecr/ts"),
Config: map[string]string{
"aws:region": region,
},
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
r, ok := stack.Outputs["ref"].(string)
require.True(t, ok)
assert.NotEmpty(t, r)
ref, err := ref.New(r)
require.NoError(t, err)
assert.NotEmpty(t, ref.Digest)
},
})

integration.ProgramTest(t, &test)
}

func TestDigitaloceanContainerRegistry(t *testing.T) {
t.Skipf("Skipping test due to known storageUsageBytes issue https://github.com/pulumi/pulumi-docker/issues/718")

Expand Down
5 changes: 3 additions & 2 deletions examples/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/aws/aws-sdk-go v1.49.0
github.com/docker/docker v24.0.7+incompatible
github.com/pulumi/pulumi/pkg/v3 v3.107.0
github.com/regclient/regclient v0.5.7
github.com/stretchr/testify v1.8.4
)

Expand Down Expand Up @@ -80,7 +81,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.4.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
Expand Down Expand Up @@ -156,7 +157,7 @@ require (
github.com/segmentio/encoding v0.3.5 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/skeema/knownhosts v1.2.1 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect
Expand Down
12 changes: 7 additions & 5 deletions examples/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -832,7 +832,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
Expand Down Expand Up @@ -1207,8 +1207,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
Expand Down Expand Up @@ -1838,6 +1838,8 @@ github.com/pulumi/pulumi/sdk/v3 v3.107.0 h1:bef+ayh9+4KkAqXih4EjlHfQXRY24NWPwWBI
github.com/pulumi/pulumi/sdk/v3 v3.107.0/go.mod h1:Ml3rpGfyZlI4zQCG7LN2XDSmH4XUNYdyBwJ3yEr/OpI=
github.com/rakyll/embedmd v0.0.0-20171029212350-c8060a0752a2/go.mod h1:7jOTMgqac46PZcF54q6l2hkLEG8op93fZu61KmxWDV4=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/regclient/regclient v0.5.7 h1:d6bXhvz7UYJM+r20ls60RIVdoYh/rp+PygD/dIsJ9UA=
github.com/regclient/regclient v0.5.7/go.mod h1:5QTWmekWy6+gq13Z6U69zsMQV5lvgxg2T7AefHT6BmA=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
Expand Down Expand Up @@ -1916,8 +1918,8 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
Expand Down
2 changes: 2 additions & 0 deletions examples/test-buildx/ecr/ts/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
/bin/
/node_modules/
3 changes: 3 additions & 0 deletions examples/test-buildx/ecr/ts/Pulumi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name: buildx-ecr
runtime: nodejs
description: A minimal AWS TypeScript Pulumi program
3 changes: 3 additions & 0 deletions examples/test-buildx/ecr/ts/app/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
FROM nginx
RUN echo "<h1>Hi from Pulumi!</h1>" > \
/usr/share/nginx/html/index.html
53 changes: 53 additions & 0 deletions examples/test-buildx/ecr/ts/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import * as aws from "@pulumi/aws";
import * as docker from "@pulumi/docker";
import * as pulumi from "@pulumi/pulumi";

// Create a private ECR registry WITHOUT force delete -- our image should be
// deleted during teardown.
const repo = new aws.ecr.Repository("my-repo", {});

// Get registry info (creds and endpoint) so we can build/publish to it.
const registryInfo = repo.registryId.apply(async (id) => {
const credentials = await aws.ecr.getCredentials({ registryId: id });
const decodedCredentials = Buffer.from(
credentials.authorizationToken,
"base64"
).toString();
const [username, password] = decodedCredentials.split(":");
if (!password || !username) {
throw new Error("Invalid credentials");
}
return {
server: credentials.proxyEndpoint,
username: username,
password: password,
};
});

// Build and publish the image.
const image = new docker.buildx.Image("buildx-pushed-multi-plat", {
tags: [pulumi.interpolate`${repo.repositoryUrl}:buildx`],
push: true,
context: {
location: "app",
},
registries: [
{
address: registryInfo.server,
username: registryInfo.username,
password: registryInfo.password,
},
],
});

const notPushed = new docker.buildx.Image("buildx-not-pushed", {
tags: [pulumi.interpolate`${repo.repositoryUrl}:buildx`],
platforms: ["linux/arm64", "linux/amd64"],
push: false,
context: {
location: "app",
},
});

// Export the resulting image name
export const ref = image.ref;
11 changes: 11 additions & 0 deletions examples/test-buildx/ecr/ts/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"name": "buildx-ecr",
"devDependencies": {
"@types/node": "^14.0.0"
},
"dependencies": {
"@pulumi/aws": "^6.10.0",
"@pulumi/pulumi": "^3.0.0",
"@pulumi/random": "^4.14.0"
}
}
18 changes: 18 additions & 0 deletions examples/test-buildx/ecr/ts/tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"compilerOptions": {
"strict": true,
"outDir": "bin",
"target": "es2016",
"module": "commonjs",
"moduleResolution": "node",
"sourceMap": true,
"experimentalDecorators": true,
"pretty": true,
"noFallthroughCasesInSwitch": true,
"noImplicitReturns": true,
"forceConsistentCasingInFileNames": true
},
"files": [
"index.ts"
]
}
4 changes: 2 additions & 2 deletions provider/cmd/pulumi-resource-docker/schema.json

Large diffs are not rendered by default.

7 changes: 5 additions & 2 deletions provider/internal/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,12 @@ func (c *CacheWithOCI) Annotate(a infer.Annotator) {
`))
a.Describe(&c.ImageManifest, dedent(`
Export cache manifest as an OCI-compatible image manifest instead of a
manifest list (requires OCI media types).
manifest list. Requires "ociMediaTypes" to also be "true".
Defaults to "false".
Some registries like AWS ECR will not work with caching if this is
"false".
Defaults to "false" to match Docker's default behavior.
`))

a.SetDefault(&c.OCI, true)
Expand Down
47 changes: 41 additions & 6 deletions provider/internal/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/flags"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/image"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth/authprovider"
Expand All @@ -41,7 +40,7 @@ type Client interface {
Build(ctx provider.Context, b Build) (*client.SolveResponse, error)
BuildKitEnabled() (bool, error)
Inspect(ctx context.Context, id string) ([]descriptor.Descriptor, error)
Delete(ctx context.Context, id string) ([]image.DeleteResponse, error)
Delete(ctx context.Context, id string) error

ManifestCreate(ctx provider.Context, push bool, target string, refs ...string) error
ManifestInspect(ctx provider.Context, target string) (string, error)
Expand Down Expand Up @@ -334,7 +333,7 @@ func (c *cli) ManifestDelete(ctx provider.Context, target string) error {
return nil
}
if err != nil {
return fmt.Errorf("fetching head: %w", err)
return err
}

return nil
Expand Down Expand Up @@ -383,11 +382,47 @@ func (c *cli) Inspect(ctx context.Context, r string) ([]descriptor.Descriptor, e
return []descriptor.Descriptor{m.GetDescriptor()}, nil
}

// Delete deletes an image with the given ID.
func (c *cli) Delete(ctx context.Context, id string) ([]image.DeleteResponse, error) {
return c.Client().ImageRemove(ctx, id, types.ImageRemoveOptions{
// Delete attempts to delete an image with the given ref. Many registries don't
// support the DELETE API yet, so this operation is not guaranteed to work.
func (c *cli) Delete(ctx context.Context, r string) error {
// Attempt to delete the ref locally if it exists.
_, _ = c.Client().ImageRemove(ctx, r, types.ImageRemoveOptions{
Force: true, // Needed in case the image has multiple tags.
})

// Attempt to delete the ref remotely if it was pushed -- requires a
// digest.
ref, err := ref.New(r)
if err != nil || ref.Digest == "" {
return nil
}

rc := c.rc()

// TODO: Multi-platform manifests are left dangling on ECR.
// m, err := rc.ManifestGet(ctx, ref)
// if err != nil {
// return err
// }

// if mi, ok := m.(manifest.Indexer); ok {
// ml, err := mi.GetManifestList()
// if err != nil {
// return err
// }

// for _, mm := range ml {
// rr := ref.SetDigest(mm.Digest.String())
// err = rc.ManifestDelete(ctx, rr, regclient.WithManifestCheckReferrers())
// if err != nil {
// return err
// }
// }
// }

_ = rc.ManifestDelete(ctx, ref)

return nil
}

func normalizeReference(ref string) (reference.Named, error) {
Expand Down
3 changes: 2 additions & 1 deletion provider/internal/doc/migration.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ The `buidx.Image` will query your registries during `refresh` to ensure the expe
If any are missing a subsequent `update` will push them.

When a `buildx.Image` is deleted, it will _attempt_ to also delete any pushed tags.
Deletion of remote tags is not guaranteed, because not all registries currently support this operation (`docker.io` in particular).
Deletion of remote tags is not guaranteed because not all registries support the manifest `DELETE` API (`docker.io` in particular).
Manifests are _not_ deleted in the same way during updates -- to do so safely would require a full build to determine whether a Pulumi operation should be an update or update-replace.

Use the [`retainOnDelete: true`](https://www.pulumi.com/docs/concepts/options/retainondelete/) option if you do not want tags deleted.

Expand Down
Loading

0 comments on commit 9743352

Please sign in to comment.