Skip to content

Commit

Permalink
[test-all] add python 3.12 to buildkite
Browse files Browse the repository at this point in the history
[test-all]
  • Loading branch information
gibsondan committed Feb 15, 2024
1 parent 91bac7d commit 8e05997
Show file tree
Hide file tree
Showing 31 changed files with 204 additions and 91 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ class AvailablePythonVersion(str, Enum):
V3_9 = "3.9"
V3_10 = "3.10"
V3_11 = "3.11"
V3_12 = "3.12"

@classmethod
def get_all(cls) -> List["AvailablePythonVersion"]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ def build_helm_steps() -> List[BuildkiteStep]:
AvailablePythonVersion.V3_8,
AvailablePythonVersion.V3_9,
AvailablePythonVersion.V3_10,
AvailablePythonVersion.V3_11,
],
name="dagster-helm",
retries=2,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,9 @@ def build_celery_k8s_suite_steps() -> List[BuildkiteTopLevelStep]:
directory,
pytest_tox_factors,
always_run_if=has_helm_changes,
unsupported_python_versions=[
AvailablePythonVersion.V3_11, # mysteriously causes buildkite agents to crash
unsupported_python_versions=[ # mysteriously causes buildkite agents to crash
AvailablePythonVersion.V3_11,
AvailablePythonVersion.V3_12,
],
)

Expand Down
115 changes: 97 additions & 18 deletions .buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
AvailablePythonVersion.V3_9,
AvailablePythonVersion.V3_10,
AvailablePythonVersion.V3_11,
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
Expand All @@ -287,10 +288,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
unsupported_python_versions=[
# dependency on 3.9-incompatible extension libs
AvailablePythonVersion.V3_9,
# dagster-airflow dep
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"examples/project_fully_featured",
unsupported_python_versions=[
AvailablePythonVersion.V3_12, # duckdb
],
),
PackageSpec(
"examples/with_great_expectations",
Expand All @@ -303,6 +309,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
),
PackageSpec(
"examples/with_wandb",
unsupported_python_versions=[
# dagster-wandb dep
AvailablePythonVersion.V3_12,
],
),
# The 6 tutorials referenced in cloud onboarding cant test "source" due to dagster-cloud dep
PackageSpec(
Expand All @@ -312,6 +322,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
PackageSpec(
"examples/assets_dbt_python",
pytest_tox_factors=["pypi"],
unsupported_python_versions=[
AvailablePythonVersion.V3_12, # duckdb
],
),
PackageSpec(
"examples/assets_dynamic_partitions",
unsupported_python_versions=[
AvailablePythonVersion.V3_12, # duckdb
],
),
PackageSpec(
"examples/quickstart_aws",
Expand All @@ -331,8 +350,39 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
),
]


def _unsupported_dagster_python_versions(tox_factor: Optional[str]) -> List[AvailablePythonVersion]:
if tox_factor == "general_tests_old_protobuf":
return [AvailablePythonVersion.V3_11, AvailablePythonVersion.V3_12]

if (
tox_factor
in {
"cli_tests", # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
}
):
return [AvailablePythonVersion.V3_11]

if tox_factor in {"scheduler_tests", "definitions_tests"}:
return [AvailablePythonVersion.V3_11]

if tox_factor in {
"definitions_tests_pendulum_1",
"definitions_tests_pendulum_2",
"scheduler_tests_pendulum_1",
"scheduler_tests_pendulum_2",
"type_signature_tests",
}:
return [AvailablePythonVersion.V3_12]

return []


LIBRARY_PACKAGES_WITH_CUSTOM_CONFIG: List[PackageSpec] = [
PackageSpec("python_modules/automation"),
PackageSpec(
"python_modules/automation",
unsupported_python_versions=[AvailablePythonVersion.V3_12],
),
PackageSpec("python_modules/dagster-webserver", pytest_extra_cmds=ui_extra_cmds),
PackageSpec(
"python_modules/dagster",
Expand Down Expand Up @@ -361,23 +411,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
"launcher_tests",
"logging_tests",
],
unsupported_python_versions=(
lambda tox_factor: (
[AvailablePythonVersion.V3_11]
if (
tox_factor
in {
"general_tests_old_protobuf", # protobuf 3 not compatible with python 3.11
"cli_tests", # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885
}
)
else (
[AvailablePythonVersion.V3_8] # pendulum 3 not supported on python 3.8
if tox_factor in {"scheduler_tests", "definitions_tests"}
else []
)
)
),
unsupported_python_versions=_unsupported_dagster_python_versions,
),
PackageSpec(
"python_modules/dagster-graphql",
Expand Down Expand Up @@ -418,6 +452,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
),
PackageSpec(
"python_modules/dagster-test",
unsupported_python_versions=[
# dagster-airflow
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-dbt",
Expand All @@ -431,6 +469,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
"dbt_pydantic1",
"dbt_legacy_pydantic1",
],
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-snowflake",
Expand All @@ -449,6 +491,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
unsupported_python_versions=[
AvailablePythonVersion.V3_10,
AvailablePythonVersion.V3_11,
AvailablePythonVersion.V3_12,
],
env_vars=[
"AIRFLOW_HOME",
Expand Down Expand Up @@ -504,8 +547,33 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
pytest_extra_cmds=docker_extra_cmds,
pytest_step_dependencies=test_project_depends_fn,
),
PackageSpec(
"python_modules/libraries/dagster-duckdb",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-duckdb-pandas",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-duckdb-polars",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-duckdb-pyspark",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagster-gcp",
Expand Down Expand Up @@ -592,10 +660,21 @@ def k8s_extra_cmds(version: str, _) -> List[str]:
# Remove once https://github.com/dagster-io/dagster/issues/2511 is resolved
retries=2,
),
PackageSpec(
"python_modules/libraries/dagster-wandb",
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
"python_modules/libraries/dagstermill",
pytest_tox_factors=["papermill1", "papermill2"],
retries=2, # Workaround for flaky kernel issues
unsupported_python_versions=[
# duckdb
AvailablePythonVersion.V3_12,
],
),
PackageSpec(
".buildkite/dagster-buildkite",
Expand Down
2 changes: 0 additions & 2 deletions examples/with_great_expectations/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
"dagster",
"dagster-ge",
"great_expectations>=0.14.12", # pinned because pip is using the cached wheel for 0.13.14
# https://github.com/great-expectations/great_expectations/issues/7990
"typing_extensions<4.6.0",
],
extras_require={"dev": ["dagster-webserver", "pytest"]},
)
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,14 @@ FROM system_base AS snapshot_builder
# package. We also install `awscli` (used in Buildkite) at this stage so that it can be installed
# together with Dagster dependencies. This allows pip's dependency resolver to account for
# everything.
RUN git clone --depth 1 https://github.com/dagster-io/dagster.git
RUN git clone https://github.com/dagster-io/dagster.git

WORKDIR dagster

# Will remove this before landing

RUN git checkout origin/python312

RUN python scripts/install_dev_python_modules.py awscli \
&& pip freeze --exclude-editable > /snapshot-reqs.txt

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"3.10": 2023-08-07T143538
"3.11": 2023-08-07T143538
"3.8": 2023-08-07T143538
"3.9": 2023-08-07T143538
"3.10": 2024-02-13T230327
"3.11": 2024-02-13T230327
"3.12": 2024-02-13T230327
"3.8": 2024-02-13T230327
"3.9": 2024-02-13T230327
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@
"3.11":
docker_args:
BASE_IMAGE: python:3.11-slim
"3.12":
docker_args:
BASE_IMAGE: python:3.12-slim
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"3.10": 2023-03-13T164608
"3.11": 2023-03-13T164608
"3.7": 2023-03-13T164608
"3.8": 2023-03-13T164608
"3.9": 2023-03-13T164608
"3.10": 2024-02-14T041706
"3.11": 2024-02-14T041706
"3.12": 2024-02-14T041706
"3.8": 2024-02-14T041706
"3.9": 2024-02-14T041706
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@
"3.11":
docker_args:
BASE_IMAGE: python:3.11-slim
"3.12":
docker_args:
BASE_IMAGE: python:3.12-slim
2 changes: 1 addition & 1 deletion python_modules/automation/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"pytablereader",
"requests",
"twine==1.15.0",
"virtualenv==20.13.2",
"virtualenv==20.25.0",
"urllib3",
],
extras_require={
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ def test_successful_host_dagster_ui_from_workspace():
log_level="warning",
)

assert server_call.called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")
server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning")


@pytest.fixture
Expand Down Expand Up @@ -185,7 +185,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
log_level="warning",
)

assert server_call.called_with(
server_call.assert_called_with(
mock.ANY, host="127.0.0.1", port=DEFAULT_WEBSERVER_PORT, log_level="warning"
)

Expand All @@ -201,7 +201,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_
log_level="warning",
)

assert server_call.called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")
server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning")


def test_successful_host_dagster_ui_from_multiple_workspace_files():
Expand Down
11 changes: 11 additions & 0 deletions python_modules/dagster/dagster/_core/test_utils.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import asyncio
import os
import re
import sys
import time
import warnings
from collections import defaultdict
Expand Down Expand Up @@ -689,3 +690,13 @@ def wrapper(*args, **kwargs):
return wrapper

return decorator


def raise_exception_on_warnings():
# turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
warnings.resetwarnings()
warnings.filterwarnings("error")

if sys.version_info >= (3, 12):
# pendulum sometimes raises DeprecationWarning on python3.12
warnings.filterwarnings("ignore", category=DeprecationWarning, module="pendulum")
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import hashlib
import os
import warnings

import pytest
from dagster import (
Expand Down Expand Up @@ -50,7 +49,7 @@
build_dep_structure_snapshot_from_graph_def,
)
from dagster._core.storage.event_log.base import EventRecordsFilter
from dagster._core.test_utils import ignore_warning, instance_for_test
from dagster._core.test_utils import ignore_warning, instance_for_test, raise_exception_on_warnings
from dagster._utils import safe_tempfile_path
from dagster._utils.warnings import (
disable_dagster_warnings,
Expand All @@ -59,10 +58,7 @@

@pytest.fixture(autouse=True)
def error_on_warning():
# turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml
warnings.resetwarnings()

warnings.filterwarnings("error")
raise_exception_on_warnings()


def _all_asset_keys(result):
Expand Down
Loading

0 comments on commit 8e05997

Please sign in to comment.