Skip to content

Commit

Permalink
Add --network and --add-host to Image (#867)
Browse files Browse the repository at this point in the history
This adds `--network` and `--add-host` options since those are supported
by both builders.

(Other requests like #761 will require more thought, since that
functionality isn't supported in the v1 API we're using right now.)

Two integration tests are included to ensure the behavior with v1 and v2
builders.

While I was in here I also added `platform` as an output.

Fixes #837.
Fixes #862.
  • Loading branch information
blampe authored Jan 29, 2024
1 parent 2298155 commit d006c5a
Show file tree
Hide file tree
Showing 19 changed files with 400 additions and 35 deletions.
32 changes: 32 additions & 0 deletions examples/examples_yaml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,3 +152,35 @@ func TestUnknownsBuildOnPreviewWarnsYAML(t *testing.T) {
},
})
}

func TestBuilderVersionsYAML(t *testing.T) {
cwd, err := os.Getwd()
if !assert.NoError(t, err) {
t.FailNow()
}

t.Run("v1", func(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: path.Join(cwd, "test-builder-version", "v1"),
Quick: true,
SkipRefresh: true,
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
platform, ok := stack.Outputs["platform"]
assert.True(t, ok)
assert.NotEmpty(t, platform)
},
})
})
t.Run("v2", func(t *testing.T) {
integration.ProgramTest(t, &integration.ProgramTestOptions{
Dir: path.Join(cwd, "test-builder-version", "v2"),
Quick: true,
SkipRefresh: true,
ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
platform, ok := stack.Outputs["platform"]
assert.True(t, ok)
assert.NotEmpty(t, platform)
},
})
})
}
3 changes: 3 additions & 0 deletions examples/test-builder-version/v1/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
FROM bash AS base

RUN getent hosts metadata.google.internal # Remember to --add-host!
22 changes: 22 additions & 0 deletions examples/test-builder-version/v1/Pulumi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: builder-v1
runtime: yaml
resources:
v1Image:
type: docker:Image
properties:
imageName: docker.io/pulumi/v1-builder:latest
skipPush: true
build:
builderVersion: BuilderV1
context: .
dockerfile: Dockerfile
platform: linux/amd64
target: base
cacheFrom:
images:
- docker.io/pulumi/pulumi-base:latest
addHosts:
- metadata.google.internal:169.254.169.254
network: host
outputs:
platform: ${v1Image.platform}
3 changes: 3 additions & 0 deletions examples/test-builder-version/v2/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
FROM bash AS base

RUN getent hosts metadata.google.internal # Remember to --add-host!
22 changes: 22 additions & 0 deletions examples/test-builder-version/v2/Pulumi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: builder-v2
runtime: yaml
resources:
v2Image:
type: docker:Image
properties:
imageName: docker.io/pulumi/v2-builder:latest
skipPush: true
build:
builderVersion: BuilderBuildKit
context: .
dockerfile: Dockerfile
platform: linux/amd64
target: base
cacheFrom:
images:
- docker.io/pulumi/pulumi-base:latest
addHosts:
- metadata.google.internal:169.254.169.254
network: host
outputs:
platform: ${v2Image.platform}
15 changes: 15 additions & 0 deletions provider/cmd/pulumi-resource-docker/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -1752,6 +1752,13 @@
"docker:index/dockerBuild:DockerBuild": {
"description": "The Docker build context",
"properties": {
"addHosts": {
"type": "array",
"items": {
"type": "string"
},
"description": "Custom host-to-IP mappings to use while building (format: \"host:ip\")"
},
"args": {
"type": "object",
"additionalProperties": {
Expand All @@ -1775,6 +1782,10 @@
"type": "string",
"description": "The path to the Dockerfile to use."
},
"network": {
"type": "string",
"description": "Set the networking mode for RUN instructions"
},
"platform": {
"type": "string",
"description": "The architecture of the platform you want to build this image for, e.g. `linux/arm64`."
Expand Down Expand Up @@ -3019,6 +3030,10 @@
"type": "string",
"description": "The fully qualified image name"
},
"platform": {
"type": "string",
"description": "The image's architecture and OS"
},
"registryServer": {
"type": "string",
"description": "The name of the registry server hosting the image."
Expand Down
76 changes: 52 additions & 24 deletions provider/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ type Build struct {
Args map[string]*string
Target string
Platform string
Network string
ExtraHosts []string
BuilderVersion types.BuilderVersion
}

Expand All @@ -90,7 +92,6 @@ func (p *dockerNativeProvider) dockerBuild(ctx context.Context,
) (string, *structpb.Struct, error) {

inputs, err := plugin.UnmarshalProperties(props, plugin.MarshalOptions{KeepUnknowns: true, SkipNulls: true})

if err != nil {
return "", nil, err
}
Expand All @@ -107,12 +108,6 @@ func (p *dockerNativeProvider) dockerBuild(ctx context.Context,
if err != nil {
return "", nil, err
}
cache, err := marshalCachedImages(inputs["build"])
if err != nil {
return "", nil, err
}

build.CachedImages = cache
img.Build = build

docker, err := configureDockerClient(p.config, true)
Expand Down Expand Up @@ -234,13 +229,15 @@ func (p *dockerNativeProvider) dockerBuild(ctx context.Context,

// make the build options
opts := types.ImageBuildOptions{
Dockerfile: replaceDockerfile,
Tags: []string{img.Name}, // this should build the image locally, sans registry info
CacheFrom: img.Build.CachedImages,
BuildArgs: build.Args,
Version: build.BuilderVersion,
Platform: build.Platform,
Target: build.Target,
Dockerfile: replaceDockerfile,
Tags: []string{img.Name}, // this should build the image locally, sans registry info
CacheFrom: img.Build.CachedImages,
BuildArgs: build.Args,
Version: build.BuilderVersion,
Platform: build.Platform,
Target: build.Target,
ExtraHosts: build.ExtraHosts,
NetworkMode: build.Network,

AuthConfigs: authConfigs,
}
Expand Down Expand Up @@ -307,6 +304,7 @@ func (p *dockerNativeProvider) dockerBuild(ctx context.Context,
"baseImageName": img.Name,
"registryServer": img.Registry.Server,
"imageName": img.Name,
"platform": img.Build.Platform,
}

imageName, err := reference.ParseNormalizedNamed(img.Name)
Expand All @@ -333,7 +331,6 @@ func (p *dockerNativeProvider) dockerBuild(ctx context.Context,
_ = p.host.LogStatus(ctx, "info", urn, "Pushing Image to the registry")

authConfigBytes, err := json.Marshal(regAuth)

if err != nil {
return "", nil, fmt.Errorf("error parsing authConfig: %v", err)
}
Expand All @@ -343,7 +340,6 @@ func (p *dockerNativeProvider) dockerBuild(ctx context.Context,

// By default, we push our image with the qualified image name from the input, without extra tagging.
pushOutput, err := docker.ImagePush(ctx, img.Name, pushOpts)

if err != nil {
return "", nil, err
}
Expand Down Expand Up @@ -463,7 +459,8 @@ func (p *dockerNativeProvider) getRepoDigest(
// 4. We only return an imageID if the image in the store matches both (1.) and (2.)
func (p *dockerNativeProvider) runImageBuild(
ctx context.Context, docker *client.Client, tar io.Reader,
opts types.ImageBuildOptions, urn resource.URN, name string) (string, error) {
opts types.ImageBuildOptions, urn resource.URN, name string,
) (string, error) {
if opts.Labels == nil {
opts.Labels = make(map[string]string)
}
Expand Down Expand Up @@ -550,7 +547,8 @@ func (p *dockerNativeProvider) runImageBuild(
}

func pullDockerImage(ctx context.Context, p *dockerNativeProvider, urn resource.URN,
docker *client.Client, authConfig types.AuthConfig, cachedImage string, platform string) error {
docker *client.Client, authConfig types.AuthConfig, cachedImage string, platform string,
) error {
if cachedImage != "" {
_ = p.host.LogStatus(ctx, "info", urn, fmt.Sprintf("Pulling cached image %s", cachedImage))

Expand Down Expand Up @@ -579,7 +577,6 @@ func pullDockerImage(ctx context.Context, p *dockerNativeProvider, urn resource.
}

func marshalBuildAndApplyDefaults(b resource.PropertyValue) (Build, error) {

// build can be nil, a string or an object; we will also use reasonable defaults here.
var build Build
if b.IsNull() {
Expand Down Expand Up @@ -617,7 +614,6 @@ func marshalBuildAndApplyDefaults(b resource.PropertyValue) (Build, error) {

// BuildKit
version, err := marshalBuilder(buildObject["builderVersion"])

if err != nil {
return build, err
}
Expand All @@ -631,13 +627,48 @@ func marshalBuildAndApplyDefaults(b resource.PropertyValue) (Build, error) {
build.Target = buildObject["target"].StringValue()
}

// CacheFrom
cache, err := marshalCachedImages(b)
if err != nil {
return build, err
}
build.CachedImages = cache

// AddHosts
hosts, err := marshalExtraHosts(b)
if err != nil {
return build, err
}
build.ExtraHosts = hosts

// Network
if !buildObject["network"].IsNull() {
build.Network = buildObject["network"].StringValue()
}

// Platform
if !buildObject["platform"].IsNull() && !buildObject["platform"].ContainsUnknowns() {
build.Platform = buildObject["platform"].StringValue()
}
return build, nil
}

func marshalExtraHosts(b resource.PropertyValue) ([]string, error) {
var extraHosts []string
if b.IsNull() || b.ObjectValue()["addHosts"].IsNull() {
return extraHosts, nil
}
hosts := b.ObjectValue()["addHosts"].ArrayValue()

for _, host := range hosts {
if !host.IsString() {
continue
}
extraHosts = append(extraHosts, host.StringValue())
}
return extraHosts, nil
}

func marshalCachedImages(b resource.PropertyValue) ([]string, error) {
var cacheImages []string
if b.IsNull() {
Expand Down Expand Up @@ -718,7 +749,7 @@ func marshalBuilder(builder resource.PropertyValue) (types.BuilderVersion, error
var version types.BuilderVersion

if builder.IsNull() {
//set default
// set default
return defaultBuilder, nil
}
// verify valid input
Expand Down Expand Up @@ -882,7 +913,6 @@ func processLogLine(jm jsonmessage.JSONMessage,
info += jm.Status + " " + jm.Progress.String()
} else if jm.Stream != "" {
info += jm.Stream

} else {
info += jm.Status
}
Expand Down Expand Up @@ -912,7 +942,6 @@ func processLogLine(jm jsonmessage.JSONMessage,
}
for _, log := range resp.Logs {
info += fmt.Sprintf("%s\n", string(log.Msg))

}
for _, warn := range resp.Warnings {
info += fmt.Sprintf("%s\n", string(warn.Short))
Expand Down Expand Up @@ -949,7 +978,6 @@ func processLogLine(jm jsonmessage.JSONMessage,
// instead of the system-wide one.
// `verify` is a testing affordance and will always be true in production.
func configureDockerClient(configs map[string]string, verify bool) (*client.Client, error) {

host, isExplicitHost := configs["host"]

if !isExplicitHost {
Expand Down
16 changes: 8 additions & 8 deletions provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ func (p *dockerNativeProvider) Call(ctx context.Context, req *rpc.CallRequest) (

// Construct creates a new component resource.
func (p *dockerNativeProvider) Construct(ctx context.Context, req *rpc.ConstructRequest) (
*rpc.ConstructResponse, error) {
*rpc.ConstructResponse, error,
) {
return nil, status.Error(codes.Unimplemented, "construct is not yet implemented")
}

Expand All @@ -77,7 +78,6 @@ func (p *dockerNativeProvider) DiffConfig(ctx context.Context, req *rpc.DiffRequ

// Configure configures the resource provider with "globals" that control its behavior.
func (p *dockerNativeProvider) Configure(_ context.Context, req *rpc.ConfigureRequest) (*rpc.ConfigureResponse, error) {

config := setConfiguration(req.GetVariables())
for key, val := range config {
p.config[key] = val
Expand Down Expand Up @@ -112,7 +112,8 @@ func (p *dockerNativeProvider) Invoke(_ context.Context, req *rpc.InvokeRequest)
// StreamInvoke dynamically executes a built-in function in the provider. The result is streamed
// back as a series of messages.
func (p *dockerNativeProvider) StreamInvoke(
req *rpc.InvokeRequest, server rpc.ResourceProvider_StreamInvokeServer) error {
req *rpc.InvokeRequest, server rpc.ResourceProvider_StreamInvokeServer,
) error {
tok := req.GetTok()
return fmt.Errorf("unknown StreamInvoke token '%s'", tok)
}
Expand Down Expand Up @@ -311,7 +312,6 @@ func (p *dockerNativeProvider) Diff(ctx context.Context, req *rpc.DiffRequest) (
diff := map[string]*rpc.PropertyDiff{}
for key := range d.Adds {
diff[string(key)] = &rpc.PropertyDiff{Kind: rpc.PropertyDiff_ADD}

}
for key := range d.Deletes {
diff[string(key)] = &rpc.PropertyDiff{Kind: rpc.PropertyDiff_DELETE}
Expand Down Expand Up @@ -432,7 +432,6 @@ func (p *dockerNativeProvider) Read(ctx context.Context, req *rpc.ReadRequest) (

// Return properties as passed, since we do no reconciliation,
return &rpc.ReadResponse{Id: id, Inputs: inputs, Properties: properties}, nil

}

// Update updates an existing resource with new values.
Expand Down Expand Up @@ -488,7 +487,6 @@ func (p *dockerNativeProvider) Update(ctx context.Context, req *rpc.UpdateReques
SkipNulls: true,
},
)

if err != nil {
return nil, err
}
Expand All @@ -515,7 +513,8 @@ func (p *dockerNativeProvider) GetPluginInfo(context.Context, *pbempty.Empty) (*

// GetSchema returns the JSON-serialized schema for the provider.
func (p *dockerNativeProvider) GetSchema(ctx context.Context, req *rpc.GetSchemaRequest) (
*rpc.GetSchemaResponse, error) {
*rpc.GetSchemaResponse, error,
) {
if v := req.GetVersion(); v != 0 {
return nil, fmt.Errorf("unsupported schema version %d", v)
}
Expand Down Expand Up @@ -564,7 +563,8 @@ type contextHashAccumulator struct {
func (accumulator *contextHashAccumulator) hashPath(
filePath string,
relativeNameOfFile string,
fileMode fs.FileMode) error {
fileMode fs.FileMode,
) error {
hash := sha256.New()

if fileMode.Type() == fs.ModeSymlink {
Expand Down
Loading

0 comments on commit d006c5a

Please sign in to comment.