[test-all] add python 3.12 to buildkite

[test-all]

.buildkite/dagster-buildkite/dagster_buildkite/python_version.py

@@ -11,6 +11,7 @@ class AvailablePythonVersion(str, Enum):
     V3_9 = "3.9"
     V3_10 = "3.10"
     V3_11 = "3.11"
+    V3_12 = "3.12"
 
     @classmethod
     def get_all(cls) -> List["AvailablePythonVersion"]:

.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py

@@ -23,6 +23,7 @@ def build_helm_steps() -> List[BuildkiteStep]:
             AvailablePythonVersion.V3_8,
             AvailablePythonVersion.V3_9,
             AvailablePythonVersion.V3_10,
+            AvailablePythonVersion.V3_11,
         ],
         name="dagster-helm",
         retries=2,

.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py

@@ -272,6 +272,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
             AvailablePythonVersion.V3_9,
             AvailablePythonVersion.V3_10,
             AvailablePythonVersion.V3_11,
+            AvailablePythonVersion.V3_12,
         ],
     ),
     PackageSpec(
@@ -287,10 +288,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         unsupported_python_versions=[
             # dependency on 3.9-incompatible extension libs
             AvailablePythonVersion.V3_9,
+            # dagster-airflow dep
+            AvailablePythonVersion.V3_12,
         ],
     ),
     PackageSpec(
         "examples/project_fully_featured",
+        unsupported_python_versions=[
+            AvailablePythonVersion.V3_12,  # duckdb
+        ],
     ),
     PackageSpec(
         "examples/with_great_expectations",
@@ -303,6 +309,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     ),
     PackageSpec(
         "examples/with_wandb",
+        unsupported_python_versions=[
+            # dagster-wandb dep
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     # The 6 tutorials referenced in cloud onboarding cant test "source" due to dagster-cloud dep
     PackageSpec(
@@ -312,6 +322,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     PackageSpec(
         "examples/assets_dbt_python",
         pytest_tox_factors=["pypi"],
+        unsupported_python_versions=[
+            AvailablePythonVersion.V3_12,  # duckdb
+        ],
+    ),
+    PackageSpec(
+        "examples/assets_dynamic_partitions",
+        unsupported_python_versions=[
+            AvailablePythonVersion.V3_12,  # duckdb
+        ],
     ),
     PackageSpec(
         "examples/quickstart_aws",
@@ -331,8 +350,39 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     ),
 ]
 
+
+def _unsupported_dagster_python_versions(tox_factor: Optional[str]) -> List[AvailablePythonVersion]:
+    if tox_factor == "general_tests_old_protobuf":
+        return [AvailablePythonVersion.V3_11, AvailablePythonVersion.V3_12]
+
+    if (
+        tox_factor
+        in {
+            "cli_tests",  # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
+        }
+    ):
+        return [AvailablePythonVersion.V3_11]
+
+    if tox_factor in {"scheduler_tests", "definitions_tests"}:
+        return [AvailablePythonVersion.V3_11]
+
+    if tox_factor in {
+        "definitions_tests_pendulum_1",
+        "definitions_tests_pendulum_2",
+        "scheduler_tests_pendulum_1",
+        "scheduler_tests_pendulum_2",
+        "type_signature_tests",
+    }:
+        return [AvailablePythonVersion.V3_12]
+
+    return []
+
+
 LIBRARY_PACKAGES_WITH_CUSTOM_CONFIG: List[PackageSpec] = [
-    PackageSpec("python_modules/automation"),
+    PackageSpec(
+        "python_modules/automation",
+        unsupported_python_versions=[AvailablePythonVersion.V3_12],
+    ),
     PackageSpec("python_modules/dagster-webserver", pytest_extra_cmds=ui_extra_cmds),
     PackageSpec(
         "python_modules/dagster",
@@ -361,23 +411,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
             "launcher_tests",
             "logging_tests",
         ],
-        unsupported_python_versions=(
-            lambda tox_factor: (
-                [AvailablePythonVersion.V3_11]
-                if (
-                    tox_factor
-                    in {
-                        "general_tests_old_protobuf",  # protobuf 3 not compatible with python 3.11
-                        "cli_tests",  # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
-                    }
-                )
-                else (
-                    [AvailablePythonVersion.V3_8]  # pendulum 3 not supported on python 3.8
-                    if tox_factor in {"scheduler_tests", "definitions_tests"}
-                    else []
-                )
-            )
-        ),
+        unsupported_python_versions=_unsupported_dagster_python_versions,
     ),
     PackageSpec(
         "python_modules/dagster-graphql",
@@ -418,6 +452,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     ),
     PackageSpec(
         "python_modules/dagster-test",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         "python_modules/libraries/dagster-dbt",
@@ -431,6 +469,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
             "dbt_pydantic1",
             "dbt_legacy_pydantic1",
         ],
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         "python_modules/libraries/dagster-snowflake",
@@ -449,6 +491,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         unsupported_python_versions=[
             AvailablePythonVersion.V3_10,
             AvailablePythonVersion.V3_11,
+            AvailablePythonVersion.V3_12,
         ],
         env_vars=[
             "AIRFLOW_HOME",
@@ -504,8 +547,33 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         pytest_extra_cmds=docker_extra_cmds,
         pytest_step_dependencies=test_project_depends_fn,
     ),
+    PackageSpec(
+        "python_modules/libraries/dagster-duckdb",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
+    PackageSpec(
+        "python_modules/libraries/dagster-duckdb-pandas",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
+    PackageSpec(
+        "python_modules/libraries/dagster-duckdb-polars",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
     PackageSpec(
         "python_modules/libraries/dagster-duckdb-pyspark",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         "python_modules/libraries/dagster-gcp",
@@ -592,10 +660,21 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         # Remove once https://github.com/dagster-io/dagster/issues/2511 is resolved
         retries=2,
     ),
+    PackageSpec(
+        "python_modules/libraries/dagster-wandb",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
     PackageSpec(
         "python_modules/libraries/dagstermill",
         pytest_tox_factors=["papermill1", "papermill2"],
         retries=2,  # Workaround for flaky kernel issues
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         ".buildkite/dagster-buildkite",

examples/assets_dbt_python/tox.ini

@@ -4,8 +4,8 @@ skipsdist = true
 [testenv]
 download = true
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
-; note: "source" does not work at this time due to dagster-cloud source access
 install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
+; note: "source" does not work at this time due to dagster-cloud source access
 deps =
   source: -e ../../python_modules/dagster[test]
   source: -e ../../python_modules/dagster-pipes

examples/assets_modern_data_stack/tox.ini

@@ -4,8 +4,8 @@ skipsdist = true
 [testenv]
 download = True
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
-; note: "source" does not work at this time due to dagster-cloud source access
 install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
+; note: "source" does not work at this time due to dagster-cloud source access
 deps =
   source: -e ../../python_modules/dagster[test]
   source: -e ../../python_modules/dagster-pipes

examples/quickstart_aws/tox.ini

@@ -4,8 +4,8 @@ skipsdist = true
 [testenv]
 download = true
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
-; note: "source" does not work at this time due to dagster-cloud source access
 install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
+; note: "source" does not work at this time due to dagster-cloud source access
 deps =
   source: -e ../../python_modules/dagster[test]
   source: -e ../../python_modules/dagster-pipes

examples/quickstart_etl/tox.ini

@@ -4,8 +4,8 @@ skipsdist = true
 [testenv]
 download = true
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
-; note: "source" does not work at this time due to dagster-cloud source access
 install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
+; note: "source" does not work at this time due to dagster-cloud source access
 deps =
   source: -e ../../python_modules/dagster[test]
   source: -e ../../python_modules/dagster-pipes

examples/quickstart_gcp/tox.ini

@@ -4,8 +4,8 @@ skipsdist = true
 [testenv]
 download = true
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
-; note: "source" does not work at this time due to dagster-cloud source access
 install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
+; note: "source" does not work at this time due to dagster-cloud source access
 deps =
   source: -e ../../python_modules/dagster[test]
   source: -e ../../python_modules/dagster-pipes

examples/quickstart_snowflake/tox.ini

@@ -4,8 +4,8 @@ skipsdist = true
 [testenv]
 download = true
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
-; note: "source" does not work at this time due to dagster-cloud source access
 install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
+; note: "source" does not work at this time due to dagster-cloud source access
 deps =
   source: -e ../../python_modules/dagster[test]
   source: -e ../../python_modules/dagster-pipes

examples/with_great_expectations/setup.py

@@ -7,8 +7,6 @@
         "dagster",
         "dagster-ge",
         "great_expectations>=0.14.12",  # pinned because pip is using the cached wheel for 0.13.14
-        # https://github.com/great-expectations/great_expectations/issues/7990
-        "typing_extensions<4.6.0",
     ],
     extras_require={"dev": ["dagster-webserver", "pytest"]},
 )

python_modules/automation/automation/docker/images/buildkite-test/Dockerfile

@@ -157,10 +157,15 @@ FROM system_base AS snapshot_builder
 # package. We also install `awscli` (used in Buildkite) at this stage so that it can be installed
 # together with Dagster dependencies. This allows pip's dependency resolver to account for
 # everything.
-RUN git clone --depth 1 https://github.com/dagster-io/dagster.git
+RUN git clone https://github.com/dagster-io/dagster.git
 
 WORKDIR dagster
 
+RUN git checkout origin/python312
+
+# RUN python scripts/install_dev_python_modules.py awscli \
+#     && pip freeze --exclude-editable | grep -v dagster_buildkite > /snapshot-reqs.txt
+
 RUN python scripts/install_dev_python_modules.py awscli \
     && pip freeze --exclude-editable > /snapshot-reqs.txt
 
@@ -174,5 +179,7 @@ COPY --from=snapshot_builder /snapshot-reqs.txt .
 RUN pip install -r /snapshot-reqs.txt \
     && rm /snapshot-reqs.txt
 
+# RUN pip install virtualenv==20.25.0 platformdirs==3.11.0
+
 # New versions of debian require allowlisting folders where `git` can run
 RUN git config --system --add safe.directory \*

python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml

@@ -1,4 +1,5 @@
-"3.10": 2023-08-07T143538
-"3.11": 2023-08-07T143538
-"3.8": 2023-08-07T143538
-"3.9": 2023-08-07T143538
+"3.10": 2024-02-05T021632
+"3.11": 2024-02-05T021632
+"3.12": 2024-02-05T021632
+"3.8": 2024-02-05T021632
+"3.9": 2024-02-05T021632

python_modules/automation/automation/docker/images/buildkite-test/versions.yaml

@@ -10,3 +10,6 @@
 "3.11":
   docker_args:
     BASE_IMAGE: python:3.11-slim
+"3.12":
+  docker_args:
+    BASE_IMAGE: python:3.12-slim

python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml

@@ -1,5 +1,5 @@
-"3.10": 2023-03-13T164608
-"3.11": 2023-03-13T164608
-"3.7": 2023-03-13T164608
-"3.8": 2023-03-13T164608
-"3.9": 2023-03-13T164608
+"3.10": 2024-02-04T193512
+"3.11": 2024-02-04T193512
+"3.12": 2024-02-04T193512
+"3.8": 2024-02-04T193512
+"3.9": 2024-02-04T193512

python_modules/automation/automation/docker/images/test-project-base/versions.yaml

@@ -10,3 +10,6 @@
 "3.11":
   docker_args:
     BASE_IMAGE: python:3.11-slim
+"3.12":
+  docker_args:
+    BASE_IMAGE: python:3.12-slim

python_modules/automation/setup.py

@@ -24,7 +24,7 @@
         "pytablereader",
         "requests",
         "twine==1.15.0",
-        "virtualenv==20.13.2",
+        "virtualenv==20.25.0",
         "urllib3",
     ],
     extras_require={

python_modules/dagster-graphql/tox.ini

@@ -12,6 +12,7 @@ deps =
   -e ../dagster-pipes
   postgres: -e ../libraries/dagster-postgres
   -e .
+install_command = python -m pip install {opts} {packages} --config-settings editable_mode=compat
 allowlist_externals =
   /bin/bash
 commands =

python_modules/dagster-webserver/dagster_webserver_tests/test_app.py

@@ -152,7 +152,7 @@ def test_successful_host_dagster_ui_from_workspace():
                 log_level="warning",
             )
 
-        assert server_call.called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")
+        server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")
 
 
 @pytest.fixture
@@ -185,7 +185,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
                 log_level="warning",
             )
 
-        assert server_call.called_with(
+        server_call.assert_called_with(
             mock.ANY, host="127.0.0.1", port=DEFAULT_WEBSERVER_PORT, log_level="warning"
         )
 
@@ -201,7 +201,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
                 log_level="warning",
             )
 
-        assert server_call.called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")
+        server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")
 
 
 def test_successful_host_dagster_ui_from_multiple_workspace_files():