pulumi · danielrbradley · May 30, 2024 · May 16, 2024 · May 16, 2024 · May 16, 2024
diff --git a/.pulumi.version b/.pulumi.version
@@ -0,0 +1 @@
+3.116.1
diff --git a/Makefile b/Makefile
@@ -17,7 +17,6 @@ WORKING_DIR      := $(shell pwd)
 EXAMPLES_DIR     := ${WORKING_DIR}/examples/yaml
 TESTPARALLELISM  := 4
 
-PULUMI           := bin/pulumi
 GOGLANGCILINT    := bin/golangci-lint
 
 # Override during CI using `make [TARGET] PROVIDER_VERSION=""` or by setting a PROVIDER_VERSION environment variable
@@ -26,6 +25,8 @@ PROVIDER_VERSION ?= 1.0.0-alpha.0+dev
 # Use this normalised version everywhere rather than the raw input to ensure consistency.
 VERSION_GENERIC = $(shell pulumictl convert-version --language generic --version "$(PROVIDER_VERSION)")
 
+export PULUMI_IGNORE_AMBIENT_PLUGINS = true
+
 .PHONY: ensure
 ensure:: tidy lint test_provider examples
 
@@ -60,29 +61,30 @@ examples/yaml:
 	rm -rf ${WORKING_DIR}/examples/yaml/app
 	cp -r ${WORKING_DIR}/examples/app ${WORKING_DIR}/examples/yaml/app
 
-examples/go: ${PULUMI} bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
+examples/go: .pulumi/bin/pulumi bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
 	$(call example,go)
 	@git checkout examples/go/go.mod
 
-examples/nodejs: ${PULUMI} bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
+examples/nodejs: .pulumi/bin/pulumi bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
 	$(call example,nodejs)
 	@git checkout examples/nodejs/package.json
 
-examples/python: ${PULUMI} bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
+examples/python: .pulumi/bin/pulumi bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
 	$(call example,python)
 	@git checkout examples/python/requirements.txt
 
-examples/dotnet: ${PULUMI} bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
+examples/dotnet: .pulumi/bin/pulumi bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
 	$(call example,dotnet)
 	@git checkout examples/dotnet/provider-docker-build.csproj
 
-examples/java: ${PULUMI} bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
+examples/java: .pulumi/bin/pulumi bin/${PROVIDER} ${WORKING_DIR}/examples/yaml/Pulumi.yaml
 	$(call example,java)
 	@git checkout examples/java/pom.xml
 
-${PULUMI}: go.sum
-	GOBIN=${WORKING_DIR}/bin go install github.com/pulumi/pulumi/pkg/v3/cmd/pulumi
-	GOBIN=${WORKING_DIR}/bin go install github.com/pulumi/pulumi/sdk/go/pulumi-language-go/v3
+.pulumi/bin/pulumi: PULUMI_VERSION := $(shell cat .pulumi.version)
+.pulumi/bin/pulumi: HOME := $(WORKING_DIR)
+.pulumi/bin/pulumi: .pulumi.version
+	curl -fsSL https://get.pulumi.com | sh -s -- --version "$(PULUMI_VERSION)"
 
 ${GOGLANGCILINT}: go.sum
 	GOBIN=${WORKING_DIR}/bin go install github.com/golangci/golangci-lint/cmd/golangci-lint
@@ -94,7 +96,7 @@ endef
 
 define example
 	rm -rf ${WORKING_DIR}/examples/$(1)
-	$(PULUMI) convert \
+	.pulumi/bin/pulumi convert \
 		--cwd ${WORKING_DIR}/examples/yaml \
 		--logtostderr \
 		--generate-only \
@@ -192,62 +194,55 @@ go.sum: go.mod
 sdk: $(shell mkdir -p sdk)
 sdk: sdk/python sdk/nodejs sdk/java sdk/python sdk/go sdk/dotnet
 
-sdk/python: PYPI_VERSION := $(shell pulumictl convert-version --language python -v "$(VERSION_GENERIC)")
 sdk/python: TMPDIR := $(shell mktemp -d)
-sdk/python: $(PULUMI) bin/${PROVIDER}
+sdk/python: .pulumi/bin/pulumi bin/${PROVIDER}
 	rm -rf sdk/python
-	$(PULUMI) package gen-sdk bin/$(PROVIDER) --language python -o ${TMPDIR}
+	.pulumi/bin/pulumi package gen-sdk bin/$(PROVIDER) --language python -o ${TMPDIR}
 	cp README.md ${TMPDIR}/python/
 	cd ${TMPDIR}/python/ && \
 		rm -rf ./bin/ ../python.bin/ && cp -R . ../python.bin && mv ../python.bin ./bin && \
-		sed -i.bak -e 's/^  version = .*/  version = "$(PYPI_VERSION)"/g' ./bin/pyproject.toml && \
-		rm ./bin/pyproject.toml.bak && \
 		python3 -m venv venv && \
 		./venv/bin/python -m pip install build && \
 		cd ./bin && \
 		../venv/bin/python -m build .
 	mv -f ${TMPDIR}/python ${WORKING_DIR}/sdk/.
 
-sdk/nodejs: NODE_VERSION := $(shell pulumictl convert-version --language javascript -v "$(VERSION_GENERIC)")
 sdk/nodejs: TMPDIR := $(shell mktemp -d)
-sdk/nodejs: $(PULUMI) bin/${PROVIDER}
+sdk/nodejs: .pulumi/bin/pulumi bin/${PROVIDER}
 	rm -rf sdk/nodejs
-	$(PULUMI) package gen-sdk bin/$(PROVIDER) --language nodejs -o ${TMPDIR}
+	.pulumi/bin/pulumi package gen-sdk bin/$(PROVIDER) --language nodejs -o ${TMPDIR}
 	cp README.md LICENSE ${TMPDIR}/nodejs
 	cd ${TMPDIR}/nodejs/ && \
 		yarn install && \
 		yarn run tsc && \
-		cp README.md LICENSE package.json yarn.lock bin/ && \
-		sed -i.bak 's/$${VERSION}/$(NODE_VERSION)/g' bin/package.json && \
-		rm ./bin/package.json.bak
+		cp README.md LICENSE package.json yarn.lock bin/
 	mv -f ${TMPDIR}/nodejs ${WORKING_DIR}/sdk/.
 
 sdk/go: TMPDIR := $(shell mktemp -d)
 sdk/go: PATH := "$(WORKING_DIR)/bin:$(PATH)"
-sdk/go: $(PULUMI) bin/${PROVIDER}
+sdk/go: .pulumi/bin/pulumi bin/${PROVIDER}
 	rm -rf sdk/go
-	PATH=$(PATH) $(PULUMI) package gen-sdk bin/$(PROVIDER) --language go -o ${TMPDIR}
+	PATH=$(PATH) .pulumi/bin/pulumi package gen-sdk bin/$(PROVIDER) --language go -o ${TMPDIR}
 	cp go.mod ${TMPDIR}/go/dockerbuild/go.mod
 	cd ${TMPDIR}/go/dockerbuild && \
 		go mod edit -module=github.com/pulumi/pulumi-${PACK}/${PACKDIR}/go/dockerbuild && \
 		go mod tidy
 	mv -f ${TMPDIR}/go ${WORKING_DIR}/sdk/go
 
-sdk/dotnet: DOTNET_VERSION  := $(shell pulumictl convert-version --language dotnet -v "$(VERSION_GENERIC)")
 sdk/dotnet: TMPDIR := $(shell mktemp -d)
-sdk/dotnet: $(PULUMI) bin/${PROVIDER}
+sdk/dotnet: .pulumi/bin/pulumi bin/${PROVIDER}
 	rm -rf sdk/dotnet
-	$(PULUMI) package gen-sdk bin/${PROVIDER} --language dotnet -o ${TMPDIR}
+	.pulumi/bin/pulumi package gen-sdk bin/${PROVIDER} --language dotnet -o ${TMPDIR}
 	cd ${TMPDIR}/dotnet/ && \
-		echo "$(DOTNET_VERSION)" > version.txt && \
-		dotnet build /p:Version=${DOTNET_VERSION}
+		echo "$(VERSION_GENERIC)" > version.txt && \
+		dotnet build
 	mv -f ${TMPDIR}/dotnet ${WORKING_DIR}/sdk/.
 
 sdk/java: PACKAGE_VERSION := $(shell pulumictl convert-version --language generic -v "$(VERSION_GENERIC)")
 sdk/java: TMPDIR := $(shell mktemp -d)
-sdk/java: $(PULUMI) bin/${PROVIDER}
+sdk/java: .pulumi/bin/pulumi bin/${PROVIDER}
 	rm -rf sdk/java
-	$(PULUMI) package gen-sdk --language java bin/${PROVIDER} -o ${TMPDIR}
+	.pulumi/bin/pulumi package gen-sdk --language java bin/${PROVIDER} -o ${TMPDIR}
 	cd ${TMPDIR}/java/ && gradle --console=plain build
 	mv -f ${TMPDIR}/java ${WORKING_DIR}/sdk/.
 

diff --git a/provider/cmd/pulumi-resource-docker-build/schema.json b/provider/cmd/pulumi-resource-docker-build/schema.json
@@ -1,7 +1,7 @@
 {
   "name": "docker-build",
   "displayName": "docker-build",
-  "version": "0.0.2-alpha.1714063884+13fb5b61",
+  "version": "1.0.0-alpha.0+dev",
   "description": "A Pulumi provider for building modern Docker images with buildx and BuildKit.",
   "keywords": [
     "docker",
@@ -20,13 +20,15 @@
     "csharp": {
       "packageReferences": {
         "Pulumi": "3.*"
-      }
+      },
+      "respectSchemaVersion": true
     },
     "go": {
       "importBasePath": "github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild",
       "packageImportAliases": {
         "github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild": "dockerbuild"
       },
+      "respectSchemaVersion": true,
       "generics": "side-by-side"
     },
     "java": {
@@ -43,12 +45,14 @@
     "nodejs": {
       "dependencies": {
         "@pulumi/pulumi": "^3.0.0"
-      }
+      },
+      "respectSchemaVersion": true
     },
     "python": {
       "requires": {
         "pulumi": "\u003e=3.0.0,\u003c4.0.0"
       },
+      "respectSchemaVersion": true,
       "pyproject": {
         "enabled": true
       }

diff --git a/provider/internal/provider.go b/provider/internal/provider.go
@@ -79,13 +79,15 @@ func NewBuildxProvider() provider.Provider {
 				LanguageMap: map[string]any{
 					"go": gogen.GoPackageInfo{
 						// GenerateResourceContainerTypes: true,
-						Generics: gogen.GenericsSettingSideBySide,
+						RespectSchemaVersion: true,
+						Generics:             gogen.GenericsSettingSideBySide,
 						PackageImportAliases: map[string]string{
 							"github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild": "dockerbuild",
 						},
 						ImportBasePath: "github.com/pulumi/pulumi-docker-build/sdk/go/dockerbuild",
 					},
 					"csharp": csgen.CSharpPackageInfo{
+						RespectSchemaVersion: true,
 						PackageReferences: map[string]string{
 							"Pulumi": "3.*",
 						},
@@ -100,11 +102,13 @@ func NewBuildxProvider() provider.Provider {
 						},
 					},
 					"nodejs": tsgen.NodePackageInfo{
+						RespectSchemaVersion: true,
 						Dependencies: map[string]string{
 							"@pulumi/pulumi": "^3.0.0",
 						},
 					},
 					"python": pygen.PackageInfo{
+						RespectSchemaVersion: true,
 						PyProject: struct {
 							Enabled bool `json:"enabled,omitempty"`
 						}{Enabled: true},

diff --git a/sdk/dotnet/Pulumi.DockerBuild.csproj b/sdk/dotnet/Pulumi.DockerBuild.csproj
diff --git a/sdk/dotnet/pulumi-plugin.json b/sdk/dotnet/pulumi-plugin.json
diff --git a/sdk/go/dockerbuild/internal/pulumiUtilities.go b/sdk/go/dockerbuild/internal/pulumiUtilities.go
diff --git a/sdk/go/dockerbuild/pulumi-plugin.json b/sdk/go/dockerbuild/pulumi-plugin.json
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		3.116.1
Copy link Contributor blampe May 16, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. @danielrbradley I don't think this is necessary if you bump the pu/pu version in `go.mod`. I'm using this provider as a bit of a testing ground to tighten up our dependency management, and getting rid of magic files like this is part of that goal. Copy link Member Author danielrbradley May 16, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. It was pulling the wrong dependency for me locally - was running a very old version of the language plugins. This is how we do it elsewhere, so standardisation seems like a good win to be able to understand different providers. Copy link Member Author danielrbradley May 17, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. If you're specifically wanting to pin codegen and pkg to the same version, you're probably better copying Ian's more complete approach which uses the proper installer: $(PULUMI): HOME := $(WORKING_DIR) $(PULUMI): provider/go.mod @ PULUMI_VERSION="$$(cd provider && go list -m github.com/pulumi/pulumi/pkg/v3 \| awk '{print $$2}')"; \ if [ -x $(PULUMI) ]; then \ CURRENT_VERSION="$$($(PULUMI) version)"; \ if [ "$${CURRENT_VERSION}" != "$${PULUMI_VERSION}" ]; then \ echo "Upgrading $(PULUMI) from $${CURRENT_VERSION} to $${PULUMI_VERSION}"; \ rm $(PULUMI); \ fi; \ fi; \ if ! [ -x $(PULUMI) ]; then \ curl -fsSL https://get.pulumi.com \| sh -s -- --version "$${PULUMI_VERSION#v}"; \ fi The downside of this is that it's a little harder to debug if it breaks compared to: .pulumi/bin/pulumi: PULUMI_VERSION := $(shell cat .pulumi.version) .pulumi/bin/pulumi: HOME := $(WORKING_DIR) .pulumi/bin/pulumi: .pulumi.version curl -fsSL https://get.pulumi.com \| sh -s -- --version "$(PULUMI_VERSION)" Additionally, we've actually needed in the past to use different versions of codegen vs pkg reference .. which was one of the driving forces for introducing `pulumi package gen-sdk`. If we're unable to upgrade pkg for any reason, we might still need to take fixes to codegen. Copy link Contributor blampe May 28, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. The downside of this is that it's a little harder to debug if it breaks compared to: The downside I see with these solutions is that it's not straightforward to test against pre-release versions. Everything needs to already be published to `get.pulumi.com`, so working with local or remote branches means awkward environment variable overrides. I 100% agree that standardization should be the goal, but I also think in this case we've converged on some unnecessary complexity. It was pulling the wrong dependency for me locally - was running a very old version of the language plugins. This is how we do it elsewhere, so standardisation seems like a good win to be able to understand different providers. I remember now that when I was initially putting this together I ran into an issue with the python language plugin requiring an `exec` script. I punted on fixing that, hence why this was using your ambient plugins. We can hack around that issue by vendoring the script for now. Here's what that looks like. If we're unable to upgrade pkg for any reason, we might still need to take fixes to codegen. I might not fully understand what you mean here, but pkg and sdk (as well as the language plugins) are still versioned separately. Copy link Member Author danielrbradley May 28, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. The downside I see with these solutions is that it's not straightforward to test against pre-release versions. Working with testing local versions of codegen is a fairly unusual case on the providers team generally, though we should have this as part of the playbook. I'd suggest an alternative approach for locally testing specific codegen changes would be to run the dev build directly and not via make at all: `~/.pulumi-dev/bin/pulumi package gen-sdk bin/pulumi-resource-xyz --language python --version 2.0.0-dev` We can hack around that issue by vendoring the script for now. From your link I see you're installing each plugin manually? e.g. `GOBIN=${WORKING_DIR}/bin go install github.com/pulumi/pulumi/sdk/nodejs/cmd/pulumi-language-nodejs/v3` This seems like it should work for the time being if you'd prefer that option, though I think it would be worth discussing deviating from using the official installer with the rest of the team as this setup looks quite unusual for any maintainers coming from other providers which we generally try to avoid. Copy link Contributor blampe May 29, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Working with testing local versions of codegen is a fairly unusual case on the providers team generally, I must be unlucky then :) though we should have this as part of the playbook. I'd suggest an alternative approach for locally testing specific codegen changes would be to run the dev build directly and not via make at all: `~/.pulumi-dev/bin/pulumi package gen-sdk bin/pulumi-resource-xyz --language python --version 2.0.0-dev` From your link I see you're installing each plugin manually? e.g. `GOBIN=${WORKING_DIR}/bin go install github.com/pulumi/pulumi/sdk/nodejs/cmd/pulumi-language-nodejs/v3` This seems like it should work for the time being if you'd prefer that option, though I think it would be worth discussing deviating from using the official installer with the rest of the team as this setup looks quite unusual for any maintainers coming from other providers which we generally try to avoid. I strongly agree with you re: deviation, but what's more unusual to me is that we're creating work for ourselves by inventing a new way to manage Go dependencies when the native toolchain should suffice. Need to use a different version locally? Use a `replace` directive. Need to upgrade your local version? Use `go get`. Need to run a CLI pinned to a particular version? Use `go run`. Need to periodically bump a version? Use a tool like Dependabot which understands `go.mod` (but not `.pulumi.version` files). A playbook would help but still has a discoverability problem. If I'm brand new to the team, why should I expect there to be two ways to manage dependencies? Copy link Member Author danielrbradley May 29, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. The purpose of downloading the CLI for codegen was to reduce our dependency on internal parts of the pulumi/pulumi codebase as these have caused issues in the past relating to breaking changes. Instead, the providers team consume the Pulumi CLI in the same way as any external users which makes compatibility easier to reason about. Installing the Pulumi CLI & language plugins via the go toolchain is not a supported installation method and is likely to break. E.g. languages being moved to their own repos. There's certainly nicities to treating it that way as you highlight, but it also requires the provider to be coupled to the internal implementation details of the CLI's build process rather than the public interface (the installer) which we advertise to our users. Additionally, almost all providers don't have a root go.mod file - this would have to be integrated into the provider module. Copy link Member mjeffryes May 29, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. it sounds like there's a decision to be made across providers about the best way to manage the pulumi version, but let's not let that block us on getting the go version support rolled out. IIUC, there's no strict dependency on avoiding ambient plugins, to start setting for `RespectSchemaVersion`? If so, I think we should just do that to move this forward. I do agree with Daniel though that if possible, I'd rather have consistency in how we pin and download pulumi CLI across providers than an ideal approach applied to just 1 or 2 providers. That ultimately makes it easier to improve all providers later. I think you two are on similar timezones this week, maybe you can get 30min to sync tomorrow and hash out a pragmatic approach? Copy link Member Author danielrbradley May 30, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Yup, keen to get this in so have adopted the change from your branch @blampe if you're happy with that. Ran a test locally and those changes fixed it for with with the extra change for `make build` which failed. I also timed the pulumi CLI installation process: Using go install: `make bin/pulumi 25.44s user 13.49s system 88% cpu 44.224 total` Using curl: `make .pulumi/bin/pulumi 2.78s user 1.43s system 36% cpu 11.469 total` Let's revisit standardisation later. Copy link Contributor blampe May 30, 2024 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. The purpose of downloading the CLI for codegen was to reduce our dependency on internal parts of the pulumi/pulumi codebase as these have caused issues in the past relating to breaking changes. Instead, the providers team consume the Pulumi CLI in the same way as any external users which makes compatibility easier to reason about. Totally makes sense. Invoking the CLI this way doesn't make us any more dependent on code internals but does make us sensitive to knowing where the code lives, as you pointed out. The difference between this binary and the published one essentially boils down to the ldflag for Version, which can be somewhat alleviated with `ReadBuildInfo`. `golangci-lint` is an example of a similar hermeticity problem. The linter and the code need to be updated together in cases when new breaking rules are added, but the linter's version is currently driven by ci-mgmt -- so workflow updates become blocked until someone takes the time to fix lint errors. It would be nice to instead let the repo decide the lint version it's compatible with, so its linter can be updated independently. One way to do that would be with a similar `.golangci-lint.version` file, but by leveraging `go.mod` instead we would have an approach that works for all tooling like this. Additionally, almost all providers don't have a root go.mod file - this would have to be integrated into the provider module. They should have a root `go.mod`, but that's a conversation for later :) I also timed the pulumi CLI installation process: Using go install: make bin/pulumi 25.44s user 13.49s system 88% cpu 44.224 total Using curl: make .pulumi/bin/pulumi 2.78s user 1.43s system 36% cpu 11.469 total Yeah, this is expected since it's compiled from source. The `tools.go` trick used here will be officially supported in go ~1.24 as `go get -tool` and should have nicer caching semantics. Anyway thanks for indulging me again, as I mentioned I'd like to spend some more time flushing out a prototype to hopefully make the benefits more obvious.