Run more tests in parallel

.github/workflows/ci.yml

@@ -124,7 +124,7 @@ jobs:
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
             ${{ env.DOCKER_ORG }}/pulumi:${{ env.PULUMI_VERSION }} \
-            -test.parallel=1 -test.timeout=1h -test.v
+            -test.timeout=1h -test.v
 
   provider-build-environment:
     name: Provider Build Environment image
@@ -194,7 +194,7 @@ jobs:
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
             ${{ env.DOCKER_ORG }}/pulumi-provider-build-environment:${{ env.PULUMI_VERSION }} \
-            -test.parallel=1 -test.timeout=1h -test.v
+            -test.timeout=1h -test.v
 
   base:
     name: Base image
@@ -317,7 +317,7 @@ jobs:
             --entrypoint /src/pulumi-test-containers \
             --platform ${{ matrix.arch }} \
             ${{ env.IMAGE_NAME }} \
-            -test.parallel=1 -test.timeout=1h -test.v -test.run "TestPulumiTemplateTests|TestEnvironment"
+            -test.timeout=1h -test.v
 
   ubi-sdk:
     name: UBI SDK images
@@ -401,4 +401,4 @@ jobs:
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
             ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi \
-            -test.parallel=1 -test.timeout=1h -test.v -test.run "TestPulumiTemplateTests|TestEnvironment"
+            -test.timeout=1h -test.v

.github/workflows/release.yml

@@ -116,7 +116,7 @@ jobs:
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
             ${{ env.DOCKER_ORG }}/pulumi:${{ env.PULUMI_VERSION }} \
-            -test.parallel=1 -test.timeout=1h -test.v
+            -test.timeout=1h -test.v
       - name: Push ${{ env.PULUMI_VERSION }}
         run: docker push ${{ env.DOCKER_ORG }}/pulumi:${{ env.PULUMI_VERSION }}
       - name: Push latest
@@ -191,7 +191,7 @@ jobs:
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
             ${{ env.DOCKER_ORG }}/pulumi-provider-build-environment:${{ env.PULUMI_VERSION }} \
-            -test.parallel=1 -test.timeout=1h -test.v
+            -test.timeout=1h -test.v
       - name: Push ${{ env.PULUMI_VERSION }}
         run: docker push ${{ env.DOCKER_ORG }}/pulumi-provider-build-environment:${{ env.PULUMI_VERSION }}
       - name: Push latest
@@ -369,7 +369,7 @@ jobs:
             --entrypoint /src/pulumi-test-containers \
             --platform ${{ matrix.arch }} \
             ${{ env.IMAGE_NAME }} \
-            -test.parallel=1 -test.timeout=1h -test.v -test.run "TestPulumiTemplateTests|TestEnvironment"
+            -test.timeout=1h -test.v
       - name: Push image
         run: |
           docker push ${{ env.IMAGE_NAME }}
@@ -528,7 +528,7 @@ jobs:
             --volume /tmp:/src \
             --entrypoint /src/pulumi-test-containers \
             ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi \
-            -test.parallel=1 -test.timeout=1h -test.v -test.run "TestPulumiTemplateTests|TestEnvironment"
+            -test.timeout=1h -test.v
       - name: Push image
         run: |
           docker push ${{ env.DOCKER_ORG }}/pulumi-${{ matrix.sdk }}:${{ env.PULUMI_VERSION }}-ubi

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- Include pyenv and Python 3.9 to 3.12 in the kitchen sink image
+  ([#232](https://github.com/pulumi/pulumi-docker-containers/pull/232))
+
 - Add per-language versions of the `pulumi/pulumi-python` image
   ([#226](https://github.com/pulumi/pulumi-docker-containers/pull/226))
 

docker/pulumi/Dockerfile

@@ -1,7 +1,4 @@
-# TODO[pulumi/pulumi-docker-containers#147]: Move back to using `python:3.9-slim`
-# as the base image when upstream issues with Debian "bookworm" are resolved.
-# For now, we continue using Debian "bullseye" as the base image.
-FROM python:3.9-slim-bullseye AS base
+FROM debian:12 AS base
 
 LABEL "repository"="https://github.com/pulumi/pulumi"
 LABEL "homepage"="https://pulumi.com"
@@ -13,16 +10,27 @@ ENV GOLANG_SHA256 b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55
 
 # Install deps all in one step
 RUN apt-get update -y && \
-  apt-get install -y \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y \
   apt-transport-https \
   build-essential \
   ca-certificates \
   curl \
   git \
   gnupg \
+  libbz2-dev \
+  libffi-dev \
+  liblzma-dev \
+  libncurses5-dev \
+  libreadline-dev \
+  libsqlite3-dev \
+  libssl-dev \
+  libxml2-dev \
+  libxmlsec1-dev \
   software-properties-common \
+  unzip  \
   wget \
-  unzip && \
+  xz-utils \
+  zlib1g-dev && \
   # Get all of the signatures we need all at once.
   curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key  | apt-key add - && \
   curl -fsSL https://dl.yarnpkg.com/debian/pubkey.gpg              | apt-key add - && \
@@ -57,6 +65,7 @@ RUN apt-get update -y && \
   google-cloud-sdk-gke-gcloud-auth-plugin \
   kubectl \
   nodejs \
+  npm \
   yarn && \
   # Clean up the lists work
   rm -rf /var/lib/apt/lists/*
@@ -96,6 +105,25 @@ RUN curl -L https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-
   helm repo add stable https://charts.helm.sh/stable && \
   helm repo update
 
+# Python
+# Install Pyenv and preinstall supported Python versions
+RUN git clone --depth=1 https://github.com/pyenv/pyenv.git /pyenv
+ENV PYENV_ROOT /pyenv
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}"
+RUN pyenv install 3.12
+RUN pyenv install 3.11
+RUN pyenv install 3.10
+RUN pyenv install 3.9
+RUN pyenv global 3.9 # Default version
+# Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="/root/.local/bin:${PATH}"
+# Set poetry to prefer the active python version so it interacts nicely with pyenv, otherwise 
+# poetry will create virtual environments using the python version used by poetry itself.
+RUN poetry config virtualenvs.prefer-active-python true
+# Enable support for .python-version files
+ENV PULUMI_LANGUAGE_VERSION_FILES=true
+
 # Passing --build-arg PULUMI_VERSION=vX.Y.Z will use that version
 # of the SDK. Otherwise, we use whatever get.pulumi.com thinks is
 # the latest

tests/containers_test.go

@@ -15,19 +15,27 @@
 package containers
 
 import (
+	"crypto/rand"
+	"embed"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/pulumi/pulumi/pkg/v3/engine"
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 	ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing"
 	"github.com/stretchr/testify/require"
 )
 
+//go:embed all:testdata
+var testdata embed.FS
+
 type testCase struct {
 	template string
 	config   map[string]string
@@ -115,7 +123,59 @@ func TestPulumiTemplateTests(t *testing.T) {
 	}
 }
 
-func TestCLIToolTests(t *testing.T) {
+func TestKitchenSinkPythonVersions(t *testing.T) {
+	if !isKitchenSink(t) {
+		t.Skip("Only running python version tests on kitchen sink")
+	}
+	t.Parallel()
+
+	t.Run("defaults to 3.9", func(t *testing.T) {
+		copyTestData(t, "testdata/python-default")
+		integration.ProgramTest(t, &integration.ProgramTestOptions{
+			Dir:         "testdata/python-default",
+			Quick:       true,
+			SkipRefresh: true,
+			PrepareProject: func(info *engine.Projinfo) error {
+				cmd := exec.Command("pulumi", "install")
+				cmd.Dir = info.Root
+				return cmd.Run()
+			},
+		})
+	})
+
+	t.Run("select 3.9", func(t *testing.T) {
+		copyTestData(t, "testdata/python-3.9")
+		integration.ProgramTest(t, &integration.ProgramTestOptions{
+			Dir:         "testdata/python-3.9",
+			Quick:       true,
+			SkipRefresh: true,
+			PrepareProject: func(info *engine.Projinfo) error {
+				cmd := exec.Command("pulumi", "install")
+				cmd.Dir = info.Root
+				return cmd.Run()
+			},
+		})
+	})
+
+	t.Run("select 3.12", func(t *testing.T) {
+		copyTestData(t, "testdata/python-3.12")
+		integration.ProgramTest(t, &integration.ProgramTestOptions{
+			Dir:         "testdata/python-3.12",
+			Quick:       true,
+			SkipRefresh: true,
+			PrepareProject: func(info *engine.Projinfo) error {
+				cmd := exec.Command("pulumi", "install")
+				cmd.Dir = info.Root
+				return cmd.Run()
+			},
+		})
+	})
+}
+
+func TestCloudCLIToolTests(t *testing.T) {
+	if !isKitchenSink(t) {
+		t.Skip("Only running Cloud CLI tool tests on kitchen sink")
+	}
 	t.Parallel()
 
 	t.Run("Azure CLI", func(t *testing.T) {
@@ -173,11 +233,14 @@ func TestEnvironment(t *testing.T) {
 	t.Logf("Testing image variant: %s", imageVariant)
 
 	t.Run("Python", func(t *testing.T) {
+		t.Parallel()
 		if !hasPython(t) {
 			t.Skip("Skipping test for images without python")
 		}
-		t.Parallel()
 		expected := "/usr/local/bin/python"
+		if isKitchenSink(t) {
+			expected = "/pyenv/shims/python"
+		}
 		if isUBI(t) {
 			expected = "/usr/bin/python"
 		}
@@ -189,14 +252,14 @@ func TestEnvironment(t *testing.T) {
 	})
 
 	t.Run("Node", func(t *testing.T) {
+		t.Parallel()
 		if !hasNodejs(t) {
 			t.Skip("Skipping test for images without nodejs")
 		}
 		expected := "/usr/local/bin/node"
 		if isUBI(t) || isKitchenSink(t) {
 			expected = "/usr/bin/node"
 		}
-		t.Parallel()
 		p, err := exec.LookPath("node")
 		require.NoError(t, err)
 		require.Equal(t, expected, p)
@@ -205,10 +268,11 @@ func TestEnvironment(t *testing.T) {
 	})
 
 	t.Run("PATH", func(t *testing.T) {
+		t.Parallel()
 		// Install scripts for various tools can sometimes modify PATH, usually by adding entries
 		// to ~/.bashrc. This test ensures that we notice such modifications.
 		expectedPaths := map[string]string{
-			"pulumi":               "/usr/share/dotnet:/pulumi/bin:/go/bin:/usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"pulumi":               "/root/.local/bin:/pyenv/shims:/pyenv/bin:/usr/share/dotnet:/pulumi/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 			"pulumi-debian-dotnet": "/root/.dotnet:/pulumi/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 			// TODO: does not include $GOPATH/bin https://github.com/pulumi/pulumi-docker-containers/issues/220
 			"pulumi-debian-go":     "/pulumi/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
@@ -327,3 +391,23 @@ func isUBI(t *testing.T) bool {
 	imageVariant := mustEnv(t, "IMAGE_VARIANT")
 	return strings.HasPrefix(imageVariant, "pulumi-ubi")
 }
+
+func RandomStackName(t *testing.T) string {
+	t.Helper()
+	b := make([]byte, 4)
+	_, err := rand.Read(b)
+	require.NoError(t, err)
+	return "test" + hex.EncodeToString(b)
+}
+
+func copyTestData(t *testing.T, path string) {
+	require.NoError(t, os.MkdirAll(path, os.ModePerm))
+	files, err := testdata.ReadDir(path)
+	require.NoError(t, err, "readdir")
+	for _, file := range files {
+		p := filepath.Join(path, file.Name())
+		fileContent, err := testdata.ReadFile(p)
+		require.NoError(t, err, "readfile")
+		require.NoError(t, os.WriteFile(p, fileContent, os.ModePerm), "writefile")
+	}
+}