Skip to content

Commit

Permalink
Add python 3.12 testing to buildkite (#19588)
Browse files Browse the repository at this point in the history
## Summary & Motivation
Leaving the default at 3.11 since not all packages are supported yet,
but with the release of universal_pathlib 0.2 today we are ready to
support 3.12 in many of the core packages. The only changes that were
required here were in tests.

resolves #17350

Test Plan: BK
  • Loading branch information
gibsondan authored Feb 15, 2024
1 parent aa011c9 commit a116c44
Show file tree
Hide file tree
Showing 30 changed files with 199 additions and 90 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ class AvailablePythonVersion(str, Enum):
V3_9 = "3.9"
V3_10 = "3.10"
V3_11 = "3.11"
V3_12 = "3.12"

@classmethod
def get_all(cls) -> List["AvailablePythonVersion"]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ def build_helm_steps() -> List[BuildkiteStep]:
AvailablePythonVersion.V3_8,
AvailablePythonVersion.V3_9,
AvailablePythonVersion.V3_10,
AvailablePythonVersion.V3_11,
],
name="dagster-helm",
retries=2,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,9 @@ def build_celery_k8s_suite_steps() -> List[BuildkiteTopLevelStep]:
directory,
pytest_tox_factors,
always_run_if=has_helm_changes,
unsupported_python_versions=[
AvailablePythonVersion.V3_11, # mysteriously causes buildkite agents to crash
unsupported_python_versions=[ # mysteriously causes buildkite agents to crash
AvailablePythonVersion.V3_11,
AvailablePythonVersion.V3_12,
],
)

Expand Down
115 changes: 97 additions & 18 deletions .buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
AvailablePythonVersion.V3_9,
AvailablePythonVersion.V3_10,
AvailablePythonVersion.V3_11,
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
Expand All @@ -287,10 +288,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
unsupported_python_versions=[
# dependency on 3.9-incompatible extension libs
AvailablePythonVersion.V3_9,
# dagster-airflow dep
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"examples/project_fully_featured",
unsupported_python_versions=[
AvailablePythonVersion.V3_12, # duckdb
],
),
PackageSpec(
"examples/with_great_expectations",
Expand All @@ -303,6 +309,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
),
PackageSpec(
"examples/with_wandb",
unsupported_python_versions=[
# dagster-wandb dep
AvailablePythonVersion.V3_12,
],
),
# The 6 tutorials referenced in cloud onboarding cant test "source" due to dagster-cloud dep
PackageSpec(
Expand All @@ -312,6 +322,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
PackageSpec(
"examples/assets_dbt_python",
pytest_tox_factors=["pypi"],
unsupported_python_versions=[
AvailablePythonVersion.V3_12, # duckdb
],
),
PackageSpec(
"examples/assets_dynamic_partitions",
unsupported_python_versions=[
AvailablePythonVersion.V3_12, # duckdb
],
),
PackageSpec(
"examples/quickstart_aws",
Expand All @@ -331,8 +350,39 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
),
]


def _unsupported_dagster_python_versions(tox_factor: Optional[str]) -> List[AvailablePythonVersion]:
if tox_factor == "general_tests_old_protobuf":
return [AvailablePythonVersion.V3_11, AvailablePythonVersion.V3_12]

if (
tox_factor
in {
"cli_tests", # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
}
):
return [AvailablePythonVersion.V3_11]

if tox_factor in {"scheduler_tests", "definitions_tests"}:
return [AvailablePythonVersion.V3_11]

if tox_factor in {
"definitions_tests_pendulum_1",
"definitions_tests_pendulum_2",
"scheduler_tests_pendulum_1",
"scheduler_tests_pendulum_2",
"type_signature_tests",
}:
return [AvailablePythonVersion.V3_12]

return []


LIBRARY_PACKAGES_WITH_CUSTOM_CONFIG: List[PackageSpec] = [
PackageSpec("python_modules/automation"),
PackageSpec(
"python_modules/automation",
unsupported_python_versions=[AvailablePythonVersion.V3_12],
),
PackageSpec("python_modules/dagster-webserver", pytest_extra_cmds=ui_extra_cmds),
PackageSpec(
"python_modules/dagster",
Expand Down Expand Up @@ -361,23 +411,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
"launcher_tests",
"logging_tests",
],
unsupported_python_versions=(
lambda tox_factor: (
[AvailablePythonVersion.V3_11]
if (
tox_factor
in {
"general_tests_old_protobuf", # protobuf 3 not compatible with python 3.11
"cli_tests", # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
}
)
else (
[AvailablePythonVersion.V3_8] # pendulum 3 not supported on python 3.8
if tox_factor in {"scheduler_tests", "definitions_tests"}
else []
)
)
),
unsupported_python_versions=_unsupported_dagster_python_versions,
),
PackageSpec(
"python_modules/dagster-graphql",
Expand Down Expand Up @@ -418,6 +452,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
),
PackageSpec(
"python_modules/dagster-test",
unsupported_python_versions=[
# dagster-airflow
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-dbt",
Expand All @@ -431,6 +469,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
"dbt_pydantic1",
"dbt_legacy_pydantic1",
],
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-snowflake",
Expand All @@ -449,6 +491,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
unsupported_python_versions=[
AvailablePythonVersion.V3_10,
AvailablePythonVersion.V3_11,
AvailablePythonVersion.V3_12,
],
env_vars=[
"AIRFLOW_HOME",
Expand Down Expand Up @@ -504,8 +547,33 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
pytest_extra_cmds=docker_extra_cmds,
pytest_step_dependencies=test_project_depends_fn,
),
PackageSpec(
"python_modules/libraries/dagster-duckdb",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-duckdb-pandas",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-duckdb-polars",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-duckdb-pyspark",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-gcp",
Expand Down Expand Up @@ -592,10 +660,21 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
# Remove once https://github.com/dagster-io/dagster/issues/2511 is resolved
retries=2,
),
PackageSpec(
"python_modules/libraries/dagster-wandb",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagstermill",
pytest_tox_factors=["papermill1", "papermill2"],
retries=2, # Workaround for flaky kernel issues
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
".buildkite/dagster-buildkite",
Expand Down
2 changes: 0 additions & 2 deletions examples/with_great_expectations/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
"dagster",
"dagster-ge",
"great_expectations>=0.14.12", # pinned because pip is using the cached wheel for 0.13.14
# https://github.com/great-expectations/great_expectations/issues/7990
"typing_extensions<4.6.0",
],
extras_require={"dev": ["dagster-webserver", "pytest"]},
)
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"3.10": 2023-08-07T143538
"3.11": 2023-08-07T143538
"3.8": 2023-08-07T143538
"3.9": 2023-08-07T143538
"3.10": 2024-02-13T230327
"3.11": 2024-02-13T230327
"3.12": 2024-02-13T230327
"3.8": 2024-02-13T230327
"3.9": 2024-02-13T230327
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@
"3.11":
docker_args:
BASE_IMAGE: python:3.11-slim
"3.12":
docker_args:
BASE_IMAGE: python:3.12-slim
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"3.10": 2023-03-13T164608
"3.11": 2023-03-13T164608
"3.7": 2023-03-13T164608
"3.8": 2023-03-13T164608
"3.9": 2023-03-13T164608
"3.10": 2024-02-14T041706
"3.11": 2024-02-14T041706
"3.12": 2024-02-14T041706
"3.8": 2024-02-14T041706
"3.9": 2024-02-14T041706
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@
"3.11":
docker_args:
BASE_IMAGE: python:3.11-slim
"3.12":
docker_args:
BASE_IMAGE: python:3.12-slim
2 changes: 1 addition & 1 deletion python_modules/automation/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"pytablereader",
"requests",
"twine==1.15.0",
"virtualenv==20.13.2",
"virtualenv==20.25.0",
"urllib3",
],
extras_require={
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def test_successful_host_dagster_ui_from_workspace():
log_level="warning",
)

assert server_call.called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")
server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")


@pytest.fixture
Expand Down Expand Up @@ -185,7 +185,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
log_level="warning",
)

assert server_call.called_with(
server_call.assert_called_with(
mock.ANY, host="127.0.0.1", port=DEFAULT_WEBSERVER_PORT, log_level="warning"
)

Expand All @@ -201,7 +201,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
log_level="warning",
)

assert server_call.called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")
server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")


def test_successful_host_dagster_ui_from_multiple_workspace_files():
Expand Down
11 changes: 11 additions & 0 deletions python_modules/dagster/dagster/_core/test_utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import asyncio
import os
import re
import sys
import time
import warnings
from collections import defaultdict
Expand Down Expand Up @@ -689,3 +690,13 @@ def wrapper(*args, **kwargs):
return wrapper

return decorator


def raise_exception_on_warnings():
# turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
warnings.resetwarnings()
warnings.filterwarnings("error")

if sys.version_info >= (3, 12):
# pendulum sometimes raises DeprecationWarning on python3.12
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pendulum")
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import hashlib
import os
import warnings

import pytest
from dagster import (
Expand Down Expand Up @@ -50,7 +49,7 @@
build_dep_structure_snapshot_from_graph_def,
)
from dagster._core.storage.event_log.base import EventRecordsFilter
from dagster._core.test_utils import ignore_warning, instance_for_test
from dagster._core.test_utils import ignore_warning, instance_for_test, raise_exception_on_warnings
from dagster._utils import safe_tempfile_path
from dagster._utils.warnings import (
disable_dagster_warnings,
Expand All @@ -59,10 +58,7 @@

@pytest.fixture(autouse=True)
def error_on_warning():
# turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
warnings.resetwarnings()

warnings.filterwarnings("error")
raise_exception_on_warnings()


def _all_asset_keys(result):
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import warnings
from typing import Any

import pytest
Expand Down Expand Up @@ -48,16 +47,13 @@
from dagster._core.definitions.policy import RetryPolicy
from dagster._core.definitions.resource_requirement import ensure_requirements_satisfied
from dagster._core.errors import DagsterInvalidConfigError
from dagster._core.test_utils import ignore_warning
from dagster._core.test_utils import ignore_warning, raise_exception_on_warnings
from dagster._core.types.dagster_type import resolve_dagster_type


@pytest.fixture(autouse=True)
def error_on_warning():
# turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
warnings.resetwarnings()

warnings.filterwarnings("error")
raise_exception_on_warnings()


def test_asset_no_decorator_args():
Expand Down
Loading

0 comments on commit a116c44

Please sign in to comment.