From 0b1760b50c70a515b886900474719ce8326d1415 Mon Sep 17 00:00:00 2001 From: gibsondan Date: Thu, 15 Feb 2024 11:06:19 -0600 Subject: [PATCH] Add python 3.12 testing to buildkite (#19588) ## Summary & Motivation Leaving the default at 3.11 since not all packages are supported yet, but with the release of universal_pathlib 0.2 today we are ready to support 3.12 in many of the core packages. The only changes that were required here were in tests. resolves #17350 Test Plan: BK --- .../dagster_buildkite/python_version.py | 1 + .../dagster_buildkite/steps/helm.py | 1 + .../dagster_buildkite/steps/integration.py | 5 +- .../dagster_buildkite/steps/packages.py | 115 +++++++++++++++--- examples/with_great_expectations/setup.py | 2 - .../images/buildkite-test/last_updated.yaml | 9 +- .../images/buildkite-test/versions.yaml | 3 + .../test-project-base/last_updated.yaml | 10 +- .../images/test-project-base/versions.yaml | 3 + python_modules/automation/setup.py | 2 +- .../dagster_webserver_tests/test_app.py | 6 +- .../dagster/dagster/_core/test_utils.py | 11 ++ .../asset_defs_tests/test_assets_job.py | 8 +- .../asset_defs_tests/test_decorators.py | 8 +- .../asset_defs_tests/test_materialize.py | 8 +- .../test_partitioned_assets.py | 8 +- .../command_tests/file_with_local_import.py | 19 ++- .../command_tests/test_cli_commands.py | 2 +- .../test_asset_execution_context.py | 7 +- .../execution_tests/test_context.py | 7 +- .../daemon_sensor_tests/conftest.py | 2 +- .../test_pythonic_resources.py | 2 +- .../test_step_delegating_executor.py | 2 + .../grpc_tests/grpc_repo_with_local_import.py | 16 ++- .../dagster_tests/scheduler_tests/conftest.py | 2 +- .../test_pythonic_resources.py | 2 +- python_modules/dagster/setup.py | 9 +- .../libraries/dagster-airflow/setup.py | 2 +- .../libraries/dagster-celery-k8s/tox.ini | 1 - scripts/install_dev_python_modules.py | 16 ++- 30 files changed, 199 insertions(+), 90 deletions(-) diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py b/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py index 15d46d4d30f54..529b59c32661c 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/python_version.py @@ -11,6 +11,7 @@ class AvailablePythonVersion(str, Enum): V3_9 = "3.9" V3_10 = "3.10" V3_11 = "3.11" + V3_12 = "3.12" @classmethod def get_all(cls) -> List["AvailablePythonVersion"]: diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py index 6bfec4b75297a..660b03eec174d 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/helm.py @@ -23,6 +23,7 @@ def build_helm_steps() -> List[BuildkiteStep]: AvailablePythonVersion.V3_8, AvailablePythonVersion.V3_9, AvailablePythonVersion.V3_10, + AvailablePythonVersion.V3_11, ], name="dagster-helm", retries=2, diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py index 6329a673ffcc6..eb6422d651c9b 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py @@ -158,8 +158,9 @@ def build_celery_k8s_suite_steps() -> List[BuildkiteTopLevelStep]: directory, pytest_tox_factors, always_run_if=has_helm_changes, - unsupported_python_versions=[ - AvailablePythonVersion.V3_11, # mysteriously causes buildkite agents to crash + unsupported_python_versions=[ # mysteriously causes buildkite agents to crash + AvailablePythonVersion.V3_11, + AvailablePythonVersion.V3_12, ], ) diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py index 254f0d58980e7..583d428d94854 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py @@ -272,6 +272,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]: AvailablePythonVersion.V3_9, AvailablePythonVersion.V3_10, AvailablePythonVersion.V3_11, + AvailablePythonVersion.V3_12, ], ), PackageSpec( @@ -287,10 +288,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]: unsupported_python_versions=[ # dependency on 3.9-incompatible extension libs AvailablePythonVersion.V3_9, + # dagster-airflow dep + AvailablePythonVersion.V3_12, ], ), PackageSpec( "examples/project_fully_featured", + unsupported_python_versions=[ + AvailablePythonVersion.V3_12, # duckdb + ], ), PackageSpec( "examples/with_great_expectations", @@ -303,6 +309,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]: ), PackageSpec( "examples/with_wandb", + unsupported_python_versions=[ + # dagster-wandb dep + AvailablePythonVersion.V3_12, + ], ), # The 6 tutorials referenced in cloud onboarding cant test "source" due to dagster-cloud dep PackageSpec( @@ -312,6 +322,15 @@ def k8s_extra_cmds(version: str, _) -> List[str]: PackageSpec( "examples/assets_dbt_python", pytest_tox_factors=["pypi"], + unsupported_python_versions=[ + AvailablePythonVersion.V3_12, # duckdb + ], + ), + PackageSpec( + "examples/assets_dynamic_partitions", + unsupported_python_versions=[ + AvailablePythonVersion.V3_12, # duckdb + ], ), PackageSpec( "examples/quickstart_aws", @@ -331,8 +350,39 @@ def k8s_extra_cmds(version: str, _) -> List[str]: ), ] + +def _unsupported_dagster_python_versions(tox_factor: Optional[str]) -> List[AvailablePythonVersion]: + if tox_factor == "general_tests_old_protobuf": + return [AvailablePythonVersion.V3_11, AvailablePythonVersion.V3_12] + + if ( + tox_factor + in { + "cli_tests", # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885 + } + ): + return [AvailablePythonVersion.V3_11] + + if tox_factor in {"scheduler_tests", "definitions_tests"}: + return [AvailablePythonVersion.V3_11] + + if tox_factor in { + "definitions_tests_pendulum_1", + "definitions_tests_pendulum_2", + "scheduler_tests_pendulum_1", + "scheduler_tests_pendulum_2", + "type_signature_tests", + }: + return [AvailablePythonVersion.V3_12] + + return [] + + LIBRARY_PACKAGES_WITH_CUSTOM_CONFIG: List[PackageSpec] = [ - PackageSpec("python_modules/automation"), + PackageSpec( + "python_modules/automation", + unsupported_python_versions=[AvailablePythonVersion.V3_12], + ), PackageSpec("python_modules/dagster-webserver", pytest_extra_cmds=ui_extra_cmds), PackageSpec( "python_modules/dagster", @@ -361,23 +411,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]: "launcher_tests", "logging_tests", ], - unsupported_python_versions=( - lambda tox_factor: ( - [AvailablePythonVersion.V3_11] - if ( - tox_factor - in { - "general_tests_old_protobuf", # protobuf 3 not compatible with python 3.11 - "cli_tests", # test suite prone to hangs on unpinned grpcio version due to https://github.com/grpc/grpc/issues/31885 - } - ) - else ( - [AvailablePythonVersion.V3_8] # pendulum 3 not supported on python 3.8 - if tox_factor in {"scheduler_tests", "definitions_tests"} - else [] - ) - ) - ), + unsupported_python_versions=_unsupported_dagster_python_versions, ), PackageSpec( "python_modules/dagster-graphql", @@ -418,6 +452,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]: ), PackageSpec( "python_modules/dagster-test", + unsupported_python_versions=[ + # dagster-airflow + AvailablePythonVersion.V3_12, + ], ), PackageSpec( "python_modules/libraries/dagster-dbt", @@ -431,6 +469,10 @@ def k8s_extra_cmds(version: str, _) -> List[str]: "dbt_pydantic1", "dbt_legacy_pydantic1", ], + unsupported_python_versions=[ + # duckdb + AvailablePythonVersion.V3_12, + ], ), PackageSpec( "python_modules/libraries/dagster-snowflake", @@ -449,6 +491,7 @@ def k8s_extra_cmds(version: str, _) -> List[str]: unsupported_python_versions=[ AvailablePythonVersion.V3_10, AvailablePythonVersion.V3_11, + AvailablePythonVersion.V3_12, ], env_vars=[ "AIRFLOW_HOME", @@ -504,8 +547,33 @@ def k8s_extra_cmds(version: str, _) -> List[str]: pytest_extra_cmds=docker_extra_cmds, pytest_step_dependencies=test_project_depends_fn, ), + PackageSpec( + "python_modules/libraries/dagster-duckdb", + unsupported_python_versions=[ + # duckdb + AvailablePythonVersion.V3_12, + ], + ), + PackageSpec( + "python_modules/libraries/dagster-duckdb-pandas", + unsupported_python_versions=[ + # duckdb + AvailablePythonVersion.V3_12, + ], + ), + PackageSpec( + "python_modules/libraries/dagster-duckdb-polars", + unsupported_python_versions=[ + # duckdb + AvailablePythonVersion.V3_12, + ], + ), PackageSpec( "python_modules/libraries/dagster-duckdb-pyspark", + unsupported_python_versions=[ + # duckdb + AvailablePythonVersion.V3_12, + ], ), PackageSpec( "python_modules/libraries/dagster-gcp", @@ -592,10 +660,21 @@ def k8s_extra_cmds(version: str, _) -> List[str]: # Remove once https://github.com/dagster-io/dagster/issues/2511 is resolved retries=2, ), + PackageSpec( + "python_modules/libraries/dagster-wandb", + unsupported_python_versions=[ + # duckdb + AvailablePythonVersion.V3_12, + ], + ), PackageSpec( "python_modules/libraries/dagstermill", pytest_tox_factors=["papermill1", "papermill2"], retries=2, # Workaround for flaky kernel issues + unsupported_python_versions=[ + # duckdb + AvailablePythonVersion.V3_12, + ], ), PackageSpec( ".buildkite/dagster-buildkite", diff --git a/examples/with_great_expectations/setup.py b/examples/with_great_expectations/setup.py index cbbbf9911d6a9..2d70c4dd0aca1 100644 --- a/examples/with_great_expectations/setup.py +++ b/examples/with_great_expectations/setup.py @@ -7,8 +7,6 @@ "dagster", "dagster-ge", "great_expectations>=0.14.12", # pinned because pip is using the cached wheel for 0.13.14 - # https://github.com/great-expectations/great_expectations/issues/7990 - "typing_extensions<4.6.0", ], extras_require={"dev": ["dagster-webserver", "pytest"]}, ) diff --git a/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml b/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml index 21e54b9ab7e12..fdff1ff9674b9 100644 --- a/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml +++ b/python_modules/automation/automation/docker/images/buildkite-test/last_updated.yaml @@ -1,4 +1,5 @@ -"3.10": 2023-08-07T143538 -"3.11": 2023-08-07T143538 -"3.8": 2023-08-07T143538 -"3.9": 2023-08-07T143538 +"3.10": 2024-02-13T230327 +"3.11": 2024-02-13T230327 +"3.12": 2024-02-13T230327 +"3.8": 2024-02-13T230327 +"3.9": 2024-02-13T230327 diff --git a/python_modules/automation/automation/docker/images/buildkite-test/versions.yaml b/python_modules/automation/automation/docker/images/buildkite-test/versions.yaml index e7ad7689af6aa..0060906c22ab8 100644 --- a/python_modules/automation/automation/docker/images/buildkite-test/versions.yaml +++ b/python_modules/automation/automation/docker/images/buildkite-test/versions.yaml @@ -10,3 +10,6 @@ "3.11": docker_args: BASE_IMAGE: python:3.11-slim +"3.12": + docker_args: + BASE_IMAGE: python:3.12-slim diff --git a/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml b/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml index 9048e1d28e326..b7ae3529385e5 100644 --- a/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml +++ b/python_modules/automation/automation/docker/images/test-project-base/last_updated.yaml @@ -1,5 +1,5 @@ -"3.10": 2023-03-13T164608 -"3.11": 2023-03-13T164608 -"3.7": 2023-03-13T164608 -"3.8": 2023-03-13T164608 -"3.9": 2023-03-13T164608 +"3.10": 2024-02-14T041706 +"3.11": 2024-02-14T041706 +"3.12": 2024-02-14T041706 +"3.8": 2024-02-14T041706 +"3.9": 2024-02-14T041706 diff --git a/python_modules/automation/automation/docker/images/test-project-base/versions.yaml b/python_modules/automation/automation/docker/images/test-project-base/versions.yaml index e7ad7689af6aa..0060906c22ab8 100644 --- a/python_modules/automation/automation/docker/images/test-project-base/versions.yaml +++ b/python_modules/automation/automation/docker/images/test-project-base/versions.yaml @@ -10,3 +10,6 @@ "3.11": docker_args: BASE_IMAGE: python:3.11-slim +"3.12": + docker_args: + BASE_IMAGE: python:3.12-slim diff --git a/python_modules/automation/setup.py b/python_modules/automation/setup.py index e2c4f293709e3..5b4a79c5f0cfa 100644 --- a/python_modules/automation/setup.py +++ b/python_modules/automation/setup.py @@ -24,7 +24,7 @@ "pytablereader", "requests", "twine==1.15.0", - "virtualenv==20.13.2", + "virtualenv==20.25.0", "urllib3", ], extras_require={ diff --git a/python_modules/dagster-webserver/dagster_webserver_tests/test_app.py b/python_modules/dagster-webserver/dagster_webserver_tests/test_app.py index 8a951626fbcee..48adf430783ce 100644 --- a/python_modules/dagster-webserver/dagster_webserver_tests/test_app.py +++ b/python_modules/dagster-webserver/dagster_webserver_tests/test_app.py @@ -152,7 +152,7 @@ def test_successful_host_dagster_ui_from_workspace(): log_level="warning", ) - assert server_call.called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning") + server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=2343, log_level="warning") @pytest.fixture @@ -185,7 +185,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_ log_level="warning", ) - assert server_call.called_with( + server_call.assert_called_with( mock.ANY, host="127.0.0.1", port=DEFAULT_WEBSERVER_PORT, log_level="warning" ) @@ -201,7 +201,7 @@ def test_host_dagster_webserver_choose_port(mock_is_port_in_use, mock_find_free_ log_level="warning", ) - assert server_call.called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning") + server_call.assert_called_with(mock.ANY, host="127.0.0.1", port=1234, log_level="warning") def test_successful_host_dagster_ui_from_multiple_workspace_files(): diff --git a/python_modules/dagster/dagster/_core/test_utils.py b/python_modules/dagster/dagster/_core/test_utils.py index 07ec911d94ed2..47d4093e5203f 100644 --- a/python_modules/dagster/dagster/_core/test_utils.py +++ b/python_modules/dagster/dagster/_core/test_utils.py @@ -1,6 +1,7 @@ import asyncio import os import re +import sys import time import warnings from collections import defaultdict @@ -689,3 +690,13 @@ def wrapper(*args, **kwargs): return wrapper return decorator + + +def raise_exception_on_warnings(): + # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml + warnings.resetwarnings() + warnings.filterwarnings("error") + + if sys.version_info >= (3, 12): + # pendulum sometimes raises DeprecationWarning on python3.12 + warnings.filterwarnings("ignore", category=DeprecationWarning, module="pendulum") diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py index d1168f18d3a2e..b2947744991d4 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py @@ -1,6 +1,5 @@ import hashlib import os -import warnings import pytest from dagster import ( @@ -50,7 +49,7 @@ build_dep_structure_snapshot_from_graph_def, ) from dagster._core.storage.event_log.base import EventRecordsFilter -from dagster._core.test_utils import ignore_warning, instance_for_test +from dagster._core.test_utils import ignore_warning, instance_for_test, raise_exception_on_warnings from dagster._utils import safe_tempfile_path from dagster._utils.warnings import ( disable_dagster_warnings, @@ -59,10 +58,7 @@ @pytest.fixture(autouse=True) def error_on_warning(): - # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml - warnings.resetwarnings() - - warnings.filterwarnings("error") + raise_exception_on_warnings() def _all_asset_keys(result): diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py index 314a15bf10c66..67c2ab3e44128 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py @@ -1,4 +1,3 @@ -import warnings from typing import Any import pytest @@ -48,16 +47,13 @@ from dagster._core.definitions.policy import RetryPolicy from dagster._core.definitions.resource_requirement import ensure_requirements_satisfied from dagster._core.errors import DagsterInvalidConfigError -from dagster._core.test_utils import ignore_warning +from dagster._core.test_utils import ignore_warning, raise_exception_on_warnings from dagster._core.types.dagster_type import resolve_dagster_type @pytest.fixture(autouse=True) def error_on_warning(): - # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml - warnings.resetwarnings() - - warnings.filterwarnings("error") + raise_exception_on_warnings() def test_asset_no_decorator_args(): diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py index ec910774e6e52..df9dda81ceca9 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py @@ -1,6 +1,5 @@ import os import pickle -import warnings from tempfile import TemporaryDirectory import pytest @@ -29,15 +28,12 @@ resource, with_resources, ) -from dagster._core.test_utils import ignore_warning, instance_for_test +from dagster._core.test_utils import ignore_warning, instance_for_test, raise_exception_on_warnings @pytest.fixture(autouse=True) def error_on_warning(): - # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml - warnings.resetwarnings() - - warnings.filterwarnings("error") + raise_exception_on_warnings() def test_basic_materialize(): diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py index 45162314975aa..e53b96f75e3f3 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py @@ -1,4 +1,3 @@ -import warnings from typing import Optional import dagster._check as check @@ -41,16 +40,13 @@ ASSET_PARTITION_RANGE_END_TAG, ASSET_PARTITION_RANGE_START_TAG, ) -from dagster._core.test_utils import assert_namedtuple_lists_equal +from dagster._core.test_utils import assert_namedtuple_lists_equal, raise_exception_on_warnings from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time @pytest.fixture(autouse=True) def error_on_warning(): - # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml - warnings.resetwarnings() - - warnings.filterwarnings("error") + raise_exception_on_warnings() def get_upstream_partitions_for_partition_range( diff --git a/python_modules/dagster/dagster_tests/cli_tests/command_tests/file_with_local_import.py b/python_modules/dagster/dagster_tests/cli_tests/command_tests/file_with_local_import.py index 68e17bb9cbf1e..70870fbe14ee6 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/command_tests/file_with_local_import.py +++ b/python_modules/dagster/dagster_tests/cli_tests/command_tests/file_with_local_import.py @@ -1,7 +1,18 @@ # type: ignore import dummy_local_file # noqa: F401 +from dagster import job, op -from dagster_tests.cli_tests.command_tests.test_cli_commands import ( - foo_job, # noqa: F401 - qux_job, # noqa: F401 -) + +@op +def do_something_op(): + return 1 + + +@op +def do_input_op(x): + return x + + +@job +def qux_job(): + do_input_op(do_something_op()) diff --git a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_cli_commands.py b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_cli_commands.py index 615fb5bc9f405..e81d0cc693540 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_cli_commands.py +++ b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_cli_commands.py @@ -688,7 +688,7 @@ def valid_job_python_origin_target_cli_args(): "-a", "bar", "-d", - os.path.dirname(__file__), + os.path.join(os.path.dirname(__file__), "..", "..", ".."), "-j", "qux", ], diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_execution_context.py b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_execution_context.py index 5c073353b2b64..a37ac36a9e17a 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_execution_context.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_execution_context.py @@ -1,8 +1,7 @@ -import warnings - import pytest from dagster import AssetExecutionContext, OpExecutionContext, asset, materialize from dagster._core.execution.context.compute import _get_deprecation_kwargs +from dagster._core.test_utils import raise_exception_on_warnings def test_doc_strings(): @@ -136,9 +135,7 @@ def assert_deprecation_messages_as_expected(received_info, expected_info): def test_instance_check(): - # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml - warnings.resetwarnings() - warnings.filterwarnings("error") + raise_exception_on_warnings() @asset def test_op_context_instance_check(context: AssetExecutionContext): diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_context.py b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_context.py index 1fd029361981b..5ba7324d99c0c 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_context.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_context.py @@ -1,5 +1,3 @@ -import warnings - import dagster._check as check import pytest from dagster import ( @@ -25,6 +23,7 @@ from dagster._core.definitions.op_definition import OpDefinition from dagster._core.errors import DagsterInvalidDefinitionError, DagsterInvariantViolationError from dagster._core.storage.dagster_run import DagsterRun +from dagster._core.test_utils import raise_exception_on_warnings def test_op_execution_context(): @@ -45,9 +44,7 @@ def foo(): def test_instance_check(): - # turn off any outer warnings filters, e.g. ignores that are set in pyproject.toml - warnings.resetwarnings() - warnings.filterwarnings("error") + raise_exception_on_warnings() class AssetExecutionContextSubclass(AssetExecutionContext): # allows us to confirm isinstance(context, AssetExecutionContext) diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/conftest.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/conftest.py index fc675da31987b..6a2a7d1b62104 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/conftest.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/conftest.py @@ -54,7 +54,7 @@ def create_workspace_load_target(attribute: Optional[str] = "the_repo") -> Modul return ModuleTarget( module_name="dagster_tests.daemon_sensor_tests.test_sensor_run", attribute=attribute, - working_directory=os.path.dirname(__file__), + working_directory=os.path.join(os.path.dirname(__file__), "..", ".."), location_name="test_location", ) diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py index 6f3af1fa63746..d2f6b1440232f 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py @@ -342,7 +342,7 @@ def create_workspace_load_target(attribute: Optional[str] = SINGLETON_REPOSITORY return ModuleTarget( module_name="dagster_tests.daemon_sensor_tests.test_pythonic_resources", attribute=None, - working_directory=os.path.dirname(__file__), + working_directory=os.path.join(os.path.dirname(__file__), "..", ".."), location_name="test_location", ) diff --git a/python_modules/dagster/dagster_tests/execution_tests/engine_tests/test_step_delegating_executor.py b/python_modules/dagster/dagster_tests/execution_tests/engine_tests/test_step_delegating_executor.py index 2d0e989fc5827..c865065b5d77d 100644 --- a/python_modules/dagster/dagster_tests/execution_tests/engine_tests/test_step_delegating_executor.py +++ b/python_modules/dagster/dagster_tests/execution_tests/engine_tests/test_step_delegating_executor.py @@ -1,3 +1,4 @@ +import os import subprocess import tempfile import threading @@ -369,6 +370,7 @@ def test_execute_using_repository_data(): recon_repo = ReconstructableRepository.for_module( "dagster_tests.execution_tests.engine_tests.test_step_delegating_executor", fn_name="pending_repo", + working_directory=os.path.join(os.path.dirname(__file__), "..", "..", ".."), ) recon_job = ReconstructableJob(repository=recon_repo, job_name="all_asset_job") diff --git a/python_modules/dagster/dagster_tests/general_tests/grpc_tests/grpc_repo_with_local_import.py b/python_modules/dagster/dagster_tests/general_tests/grpc_tests/grpc_repo_with_local_import.py index 96fa078ddcf1d..d9b9344340c50 100644 --- a/python_modules/dagster/dagster_tests/general_tests/grpc_tests/grpc_repo_with_local_import.py +++ b/python_modules/dagster/dagster_tests/general_tests/grpc_tests/grpc_repo_with_local_import.py @@ -1,3 +1,17 @@ import dummy_local_file as dummy_local_file # type: ignore +from dagster import job, op, repository -from dagster_tests.general_tests.grpc_tests.grpc_repo import bar_repo as bar_repo + +@op +def my_op(): + pass + + +@job +def my_job(): + my_op() + + +@repository +def bar_repo(): + return [my_job] diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py b/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py index 26a2584ddfbf9..0fa22c3e94722 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/conftest.py @@ -56,7 +56,7 @@ def workspace_load_target( return ModuleTarget( module_name=f"dagster_tests.scheduler_tests.{module}", attribute=attribute, - working_directory=os.path.dirname(__file__), + working_directory=os.path.join(os.path.dirname(__file__), "..", ".."), location_name="test_location", ) diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py b/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py index 77e93be77eee4..4e76d319f4b76 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py @@ -124,7 +124,7 @@ def create_workspace_load_target(attribute: Optional[str] = SINGLETON_REPOSITORY return ModuleTarget( module_name="dagster_tests.scheduler_tests.test_pythonic_resources", attribute=None, - working_directory=os.path.dirname(__file__), + working_directory=os.path.join(os.path.dirname(__file__), "..", ".."), location_name="test_location", ) diff --git a/python_modules/dagster/setup.py b/python_modules/dagster/setup.py index 781664c5e85c3..00e540b023bcb 100644 --- a/python_modules/dagster/setup.py +++ b/python_modules/dagster/setup.py @@ -86,9 +86,11 @@ def get_version() -> str: f"grpcio>={GRPC_VERSION_FLOOR}", f"grpcio-health-checking>={GRPC_VERSION_FLOOR}", "packaging>=20.9", - "pendulum>=0.7.0,<4; python_version>='3.9'", + "pendulum>=3,<4; python_version>='3.12'", + "pendulum>=0.7.0,<4; python_version>='3.9' and python_version<'3.12'", "pendulum>=0.7.0,<3; python_version<'3.9'", # https://github.com/dagster-io/dagster/issues/19500 - "protobuf>=3.20.0,<5", # min protobuf version to be compatible with both protobuf 3 and 4 + "protobuf>=3.20.0,<5; python_version<'3.11'", # min protobuf version to be compatible with both protobuf 3 and 4 + "protobuf>=4,<5; python_version>='3.11'", "python-dateutil", "python-dotenv", "pytz", @@ -106,7 +108,8 @@ def get_version() -> str: # https://github.com/mhammond/pywin32/issues/1439 'pywin32!=226; platform_system=="Windows"', "docstring-parser", - "universal_pathlib", + "universal_pathlib; python_version<'3.12'", + "universal_pathlib>=0.2.0; python_version>='3.12'", # https://github.com/pydantic/pydantic/issues/5821 "pydantic>1.10.0,!= 1.10.7,<3", "rich", diff --git a/python_modules/libraries/dagster-airflow/setup.py b/python_modules/libraries/dagster-airflow/setup.py index d1ea438d1f5a7..e306a9a29642c 100644 --- a/python_modules/libraries/dagster-airflow/setup.py +++ b/python_modules/libraries/dagster-airflow/setup.py @@ -33,8 +33,8 @@ def get_version() -> str: install_requires=[ f"dagster{pin}", "docker>=5.0.3,<6.0.0", + "urllib3<2", # docker version pinned above requires this but has no pin "lazy_object_proxy", - "pendulum<3", ], project_urls={ # airflow will embed a link this in the providers page UI diff --git a/python_modules/libraries/dagster-celery-k8s/tox.ini b/python_modules/libraries/dagster-celery-k8s/tox.ini index 729c8af49157b..4b8f4450d51a8 100644 --- a/python_modules/libraries/dagster-celery-k8s/tox.ini +++ b/python_modules/libraries/dagster-celery-k8s/tox.ini @@ -17,7 +17,6 @@ deps = -e ../dagster-celery-docker -e ../dagster-docker -e ../dagster-postgres - -e ../dagster-airflow -e ../dagster-aws -e ../dagster-gcp -e . diff --git a/scripts/install_dev_python_modules.py b/scripts/install_dev_python_modules.py index 41fd7dc072540..58a4bffe96c51 100644 --- a/scripts/install_dev_python_modules.py +++ b/scripts/install_dev_python_modules.py @@ -51,7 +51,6 @@ def main( "-e python_modules/automation", "-e python_modules/libraries/dagster-managed-elements", "-e python_modules/libraries/dagster-airbyte", - "-e python_modules/libraries/dagster-airflow", "-e python_modules/libraries/dagster-aws[test]", "-e python_modules/libraries/dagster-celery", "-e python_modules/libraries/dagster-celery-docker", @@ -86,11 +85,6 @@ def main( "-e integration_tests/python_modules/dagster-k8s-test-infra", "-e python_modules/libraries/dagster-azure", "-e python_modules/libraries/dagster-msteams", - "-e python_modules/libraries/dagster-duckdb", - "-e python_modules/libraries/dagster-duckdb-pandas", - "-e python_modules/libraries/dagster-duckdb-polars", - "-e python_modules/libraries/dagster-duckdb-pyspark", - "-e python_modules/libraries/dagster-wandb", "-e python_modules/libraries/dagster-deltalake", "-e python_modules/libraries/dagster-deltalake-pandas", "-e python_modules/libraries/dagster-deltalake-polars", @@ -98,6 +92,16 @@ def main( "-e .buildkite/dagster-buildkite", ] + if sys.version_info <= (3, 12): + install_targets += [ + "-e python_modules/libraries/dagster-duckdb", + "-e python_modules/libraries/dagster-duckdb-pandas", + "-e python_modules/libraries/dagster-duckdb-polars", + "-e python_modules/libraries/dagster-duckdb-pyspark", + "-e python_modules/libraries/dagster-wandb", + "-e python_modules/libraries/dagster-airflow", + ] + if sys.version_info > (3, 7): install_targets += [ "-e python_modules/libraries/dagster-dbt",