Add python 3.12 testing to buildkite (#19588)

## Summary & Motivation Leaving the default at 3.11 since not all packages are supported yet, but with the release of universal_pathlib 0.2 today we are ready to support 3.12 in many of the core packages. The only changes that were required here were in tests. resolves #17350 Test Plan: BK
dagster-io · Mar 28, 2024 · 0b1760b · 0b1760b
1 parent 9f12ddc
commit 0b1760b
Show file tree

Hide file tree

Showing 30 changed files with 199 additions and 90 deletions.
diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py b/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py
@@ -11,6 +11,7 @@ class AvailablePythonVersion(str, Enum):
     V3_9 = "3.9"
     V3_10 = "3.10"
     V3_11 = "3.11"
+    V3_12 = "3.12"
 
     @classmethod
     def get_all(cls) -> List["AvailablePythonVersion"]:

diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py
@@ -23,6 +23,7 @@ def build_helm_steps() -> List[BuildkiteStep]:
             AvailablePythonVersion.V3_8,
             AvailablePythonVersion.V3_9,
             AvailablePythonVersion.V3_10,
+            AvailablePythonVersion.V3_11,
         ],
         name="dagster-helm",
         retries=2,

diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py
@@ -158,8 +158,9 @@ def build_celery_k8s_suite_steps() -> List[BuildkiteTopLevelStep]:
         directory,
         pytest_tox_factors,
         always_run_if=has_helm_changes,
-        unsupported_python_versions=[
-            AvailablePythonVersion.V3_11,  # mysteriously causes buildkite agents to crash
+        unsupported_python_versions=[  # mysteriously causes buildkite agents to crash
+            AvailablePythonVersion.V3_11,
+            AvailablePythonVersion.V3_12,
         ],
     )
 

diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py
@@ -272,6 +272,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
             AvailablePythonVersion.V3_9,
             AvailablePythonVersion.V3_10,
             AvailablePythonVersion.V3_11,
+            AvailablePythonVersion.V3_12,
         ],
     ),
     PackageSpec(
@@ -287,10 +288,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         unsupported_python_versions=[
             # dependency on 3.9-incompatible extension libs
             AvailablePythonVersion.V3_9,
+            # dagster-airflow dep
+            AvailablePythonVersion.V3_12,
         ],
     ),
     PackageSpec(
         "examples/project_fully_featured",
+        unsupported_python_versions=[
+            AvailablePythonVersion.V3_12,  # duckdb
+        ],
     ),
     PackageSpec(
         "examples/with_great_expectations",
@@ -303,6 +309,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     ),
     PackageSpec(
         "examples/with_wandb",
+        unsupported_python_versions=[
+            # dagster-wandb dep
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     # The 6 tutorials referenced in cloud onboarding cant test "source" due to dagster-cloud dep
     PackageSpec(
@@ -312,6 +322,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     PackageSpec(
         "examples/assets_dbt_python",
         pytest_tox_factors=["pypi"],
+        unsupported_python_versions=[
+            AvailablePythonVersion.V3_12,  # duckdb
+        ],
+    ),
+    PackageSpec(
+        "examples/assets_dynamic_partitions",
+        unsupported_python_versions=[
+            AvailablePythonVersion.V3_12,  # duckdb
+        ],
     ),
     PackageSpec(
         "examples/quickstart_aws",
@@ -331,8 +350,39 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     ),
 ]
 
+
+def _unsupported_dagster_python_versions(tox_factor: Optional[str]) -> List[AvailablePythonVersion]:
+    if tox_factor == "general_tests_old_protobuf":
+        return [AvailablePythonVersion.V3_11, AvailablePythonVersion.V3_12]
+
+    if (
+        tox_factor
+        in {
+            "cli_tests",  # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
+        }
+    ):
+        return [AvailablePythonVersion.V3_11]
+
+    if tox_factor in {"scheduler_tests", "definitions_tests"}:
+        return [AvailablePythonVersion.V3_11]
+
+    if tox_factor in {
+        "definitions_tests_pendulum_1",
+        "definitions_tests_pendulum_2",
+        "scheduler_tests_pendulum_1",
+        "scheduler_tests_pendulum_2",
+        "type_signature_tests",
+    }:
+        return [AvailablePythonVersion.V3_12]
+
+    return []
+
+
 LIBRARY_PACKAGES_WITH_CUSTOM_CONFIG: List[PackageSpec] = [
-    PackageSpec("python_modules/automation"),
+    PackageSpec(
+        "python_modules/automation",
+        unsupported_python_versions=[AvailablePythonVersion.V3_12],
+    ),
     PackageSpec("python_modules/dagster-webserver", pytest_extra_cmds=ui_extra_cmds),
     PackageSpec(
         "python_modules/dagster",
@@ -361,23 +411,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
             "launcher_tests",
             "logging_tests",
         ],
-        unsupported_python_versions=(
-            lambda tox_factor: (
-                [AvailablePythonVersion.V3_11]
-                if (
-                    tox_factor
-                    in {
-                        "general_tests_old_protobuf",  # protobuf 3 not compatible with python 3.11
-                        "cli_tests",  # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
-                    }
-                )
-                else (
-                    [AvailablePythonVersion.V3_8]  # pendulum 3 not supported on python 3.8
-                    if tox_factor in {"scheduler_tests", "definitions_tests"}
-                    else []
-                )
-            )
-        ),
+        unsupported_python_versions=_unsupported_dagster_python_versions,
     ),
     PackageSpec(
         "python_modules/dagster-graphql",
@@ -418,6 +452,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
     ),
     PackageSpec(
         "python_modules/dagster-test",
+        unsupported_python_versions=[
+            # dagster-airflow
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         "python_modules/libraries/dagster-dbt",
@@ -431,6 +469,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
             "dbt_pydantic1",
             "dbt_legacy_pydantic1",
         ],
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         "python_modules/libraries/dagster-snowflake",
@@ -449,6 +491,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         unsupported_python_versions=[
             AvailablePythonVersion.V3_10,
             AvailablePythonVersion.V3_11,
+            AvailablePythonVersion.V3_12,
         ],
         env_vars=[
             "AIRFLOW_HOME",
@@ -504,8 +547,33 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         pytest_extra_cmds=docker_extra_cmds,
         pytest_step_dependencies=test_project_depends_fn,
     ),
+    PackageSpec(
+        "python_modules/libraries/dagster-duckdb",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
+    PackageSpec(
+        "python_modules/libraries/dagster-duckdb-pandas",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
+    PackageSpec(
+        "python_modules/libraries/dagster-duckdb-polars",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
     PackageSpec(
         "python_modules/libraries/dagster-duckdb-pyspark",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         "python_modules/libraries/dagster-gcp",
@@ -592,10 +660,21 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
         # Remove once https://github.com/dagster-io/dagster/issues/2511 is resolved
         retries=2,
     ),
+    PackageSpec(
+        "python_modules/libraries/dagster-wandb",
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
+    ),
     PackageSpec(
         "python_modules/libraries/dagstermill",
         pytest_tox_factors=["papermill1", "papermill2"],
         retries=2,  # Workaround for flaky kernel issues
+        unsupported_python_versions=[
+            # duckdb
+            AvailablePythonVersion.V3_12,
+        ],
     ),
     PackageSpec(
         ".buildkite/dagster-buildkite",

diff --git a/examples/with_great_expectations/setup.py b/examples/with_great_expectations/setup.py
@@ -7,8 +7,6 @@
         "dagster",
         "dagster-ge",
         "great_expectations>=0.14.12",  # pinned because pip is using the cached wheel for 0.13.14
-        # https://github.com/great-expectations/great_expectations/issues/7990
-        "typing_extensions<4.6.0",
     ],
     extras_require={"dev": ["dagster-webserver", "pytest"]},
 )
diff --git a/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml b/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml
@@ -1,4 +1,5 @@
-"3.10": 2023-08-07T143538
-"3.11": 2023-08-07T143538
-"3.8": 2023-08-07T143538
-"3.9": 2023-08-07T143538
+"3.10": 2024-02-13T230327
+"3.11": 2024-02-13T230327
+"3.12": 2024-02-13T230327
+"3.8": 2024-02-13T230327
+"3.9": 2024-02-13T230327
diff --git a/python_modules/automation/automation/docker/images/buildkite-test/versions.yaml b/python_modules/automation/automation/docker/images/buildkite-test/versions.yaml
@@ -10,3 +10,6 @@
 "3.11":
   docker_args:
     BASE_IMAGE: python:3.11-slim
+"3.12":
+  docker_args:
+    BASE_IMAGE: python:3.12-slim
diff --git a/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml b/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml
@@ -1,5 +1,5 @@
-"3.10": 2023-03-13T164608
-"3.11": 2023-03-13T164608
-"3.7": 2023-03-13T164608
-"3.8": 2023-03-13T164608
-"3.9": 2023-03-13T164608
+"3.10": 2024-02-14T041706
+"3.11": 2024-02-14T041706
+"3.12": 2024-02-14T041706
+"3.8": 2024-02-14T041706
+"3.9": 2024-02-14T041706
diff --git a/python_modules/automation/automation/docker/images/test-project-base/versions.yaml b/python_modules/automation/automation/docker/images/test-project-base/versions.yaml
@@ -10,3 +10,6 @@
 "3.11":
   docker_args:
     BASE_IMAGE: python:3.11-slim
+"3.12":
+  docker_args:
+    BASE_IMAGE: python:3.12-slim
diff --git a/python_modules/automation/setup.py b/python_modules/automation/setup.py
@@ -24,7 +24,7 @@
         "pytablereader",
         "requests",
         "twine==1.15.0",
-        "virtualenv==20.13.2",
+        "virtualenv==20.25.0",
         "urllib3",
     ],
     extras_require={

diff --git a/python_modules/dagster-webserver/dagster_webserver_tests/test_app.py b/python_modules/dagster-webserver/dagster_webserver_tests/test_app.py
@@ -152,7 +152,7 @@ def test_successful_host_dagster_ui_from_workspace():
                 log_level="warning",
             )
 
-        assert server_call.called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")
+        server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")
 
 
 @pytest.fixture
@@ -185,7 +185,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
                 log_level="warning",
             )
 
-        assert server_call.called_with(
+        server_call.assert_called_with(
             mock.ANY, host="127.0.0.1", port=DEFAULT_WEBSERVER_PORT, log_level="warning"
         )
 
@@ -201,7 +201,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
                 log_level="warning",
             )
 
-        assert server_call.called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")
+        server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")
 
 
 def test_successful_host_dagster_ui_from_multiple_workspace_files():

diff --git a/python_modules/dagster/dagster/_core/test_utils.py b/python_modules/dagster/dagster/_core/test_utils.py
@@ -1,6 +1,7 @@
 import asyncio
 import os
 import re
+import sys
 import time
 import warnings
 from collections import defaultdict
@@ -689,3 +690,13 @@ def wrapper(*args, **kwargs):
         return wrapper
 
     return decorator
+
+
+def raise_exception_on_warnings():
+    # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
+    warnings.resetwarnings()
+    warnings.filterwarnings("error")
+
+    if sys.version_info >= (3, 12):
+        # pendulum sometimes raises DeprecationWarning on python3.12
+        warnings.filterwarnings("ignore", category=DeprecationWarning, module="pendulum")
diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py
@@ -1,6 +1,5 @@
 import hashlib
 import os
-import warnings
 
 import pytest
 from dagster import (
@@ -50,7 +49,7 @@
     build_dep_structure_snapshot_from_graph_def,
 )
 from dagster._core.storage.event_log.base import EventRecordsFilter
-from dagster._core.test_utils import ignore_warning, instance_for_test
+from dagster._core.test_utils import ignore_warning, instance_for_test, raise_exception_on_warnings
 from dagster._utils import safe_tempfile_path
 from dagster._utils.warnings import (
     disable_dagster_warnings,
@@ -59,10 +58,7 @@
 
 @pytest.fixture(autouse=True)
 def error_on_warning():
-    # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
-    warnings.resetwarnings()
-
-    warnings.filterwarnings("error")
+    raise_exception_on_warnings()
 
 
 def _all_asset_keys(result):

diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py
@@ -1,4 +1,3 @@
-import warnings
 from typing import Any
 
 import pytest
@@ -48,16 +47,13 @@
 from dagster._core.definitions.policy import RetryPolicy
 from dagster._core.definitions.resource_requirement import ensure_requirements_satisfied
 from dagster._core.errors import DagsterInvalidConfigError
-from dagster._core.test_utils import ignore_warning
+from dagster._core.test_utils import ignore_warning, raise_exception_on_warnings
 from dagster._core.types.dagster_type import resolve_dagster_type
 
 
 @pytest.fixture(autouse=True)
 def error_on_warning():
-    # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
-    warnings.resetwarnings()
-
-    warnings.filterwarnings("error")
+    raise_exception_on_warnings()
 
 
 def test_asset_no_decorator_args():