diff --git a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py index 3b8f938f609ca..b6eff610b4d9c 100644 --- a/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py +++ b/.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py @@ -368,15 +368,17 @@ def k8s_extra_cmds(version: str, _) -> List[str]: "storage_tests_sqlalchemy_1_4", "daemon_sensor_tests", "daemon_tests", - "definitions_tests_old_pendulum", + "definitions_tests", + "definitions_tests_pendulum_1", + "definitions_tests_pendulum_2", "general_tests", "general_tests_old_protobuf", "scheduler_tests", - "scheduler_tests_old_pendulum", + "scheduler_tests_pendulum_1", + "scheduler_tests_pendulum_2", "execution_tests", "storage_tests", "type_signature_tests", - "definitions_tests", "asset_defs_tests", "launcher_tests", "logging_tests", diff --git a/integration_tests/test_suites/daemon-test-suite/test_daemon.py b/integration_tests/test_suites/daemon-test-suite/test_daemon.py index 2df20533110e5..588219711a24a 100644 --- a/integration_tests/test_suites/daemon-test-suite/test_daemon.py +++ b/integration_tests/test_suites/daemon-test-suite/test_daemon.py @@ -7,6 +7,7 @@ DEFAULT_HEARTBEAT_INTERVAL_SECONDS, all_daemons_healthy, ) +from dagster._seven.compat.pendulum import pendulum_freeze_time from utils import start_daemon @@ -23,5 +24,5 @@ def test_heartbeat(): + DEFAULT_DAEMON_HEARTBEAT_TOLERANCE_SECONDS + 5 ) - with pendulum.test(frozen_datetime): + with pendulum_freeze_time(frozen_datetime): assert all_daemons_healthy(instance) is False diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_scheduler.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_scheduler.py index f971b96d7b634..6bff50f8c6353 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_scheduler.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_scheduler.py @@ -16,7 +16,7 @@ from dagster._core.types.loadable_target_origin import LoadableTargetOrigin from dagster._core.workspace.context import WorkspaceRequestContext from dagster._seven import get_current_datetime_in_utc, get_timestamp_from_utc_datetime -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time from dagster._utils import Counter, traced_counter from dagster_graphql.implementation.utils import UserFacingGraphQLError from dagster_graphql.test.utils import ( @@ -697,7 +697,7 @@ def test_unloadable_schedule(graphql_context): stopped_origin = _get_unloadable_schedule_origin("unloadable_stopped") - with pendulum.test(initial_datetime): + with pendulum_freeze_time(initial_datetime): instance.add_instigator_state(running_instigator_state) instance.add_instigator_state( diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_sensors.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_sensors.py index bef40d152889e..b798177fadcb1 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_sensors.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_sensors.py @@ -23,6 +23,7 @@ from dagster._core.workspace.context import WorkspaceRequestContext from dagster._daemon import get_default_daemon_logger from dagster._daemon.sensor import execute_sensor_iteration +from dagster._seven.compat.pendulum import pendulum_freeze_time from dagster._utils import Counter, traced_counter from dagster._utils.error import SerializableErrorInfo from dagster_graphql.implementation.utils import UserFacingGraphQLError @@ -1062,15 +1063,15 @@ def test_sensor_tick_range(graphql_context: WorkspaceRequestContext): now = pendulum.now("US/Central") one = now.subtract(days=2).subtract(hours=1) - with pendulum.test(one): + with pendulum_freeze_time(one): _create_tick(graphql_context) two = now.subtract(days=1).subtract(hours=1) - with pendulum.test(two): + with pendulum_freeze_time(two): _create_tick(graphql_context) three = now.subtract(hours=1) - with pendulum.test(three): + with pendulum_freeze_time(three): _create_tick(graphql_context) result = execute_dagster_graphql( @@ -1174,7 +1175,7 @@ def test_sensor_ticks_filtered(graphql_context: WorkspaceRequestContext): ) now = pendulum.now("US/Central") - with pendulum.test(now): + with pendulum_freeze_time(now): _create_tick(graphql_context) # create a success tick # create a started tick diff --git a/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py b/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py index e27eae206e965..63ff5532a5d7c 100644 --- a/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py +++ b/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py @@ -10,11 +10,12 @@ import datetime from typing import TYPE_CHECKING, AbstractSet, Optional, Sequence, Tuple -import pendulum - from dagster._core.definitions.asset_subset import AssetSubset from dagster._core.definitions.events import AssetKeyPartitionKey from dagster._core.definitions.freshness_policy import FreshnessPolicy +from dagster._seven.compat.pendulum import ( + PendulumInterval, +) from dagster._utils.schedules import cron_string_iterator if TYPE_CHECKING: @@ -27,7 +28,7 @@ def get_execution_period_for_policy( freshness_policy: FreshnessPolicy, effective_data_time: Optional[datetime.datetime], current_time: datetime.datetime, -) -> pendulum.Period: +) -> PendulumInterval: if freshness_policy.cron_schedule: tick_iterator = cron_string_iterator( start_timestamp=current_time.timestamp(), @@ -41,18 +42,18 @@ def get_execution_period_for_policy( tick = next(tick_iterator) required_data_time = tick - freshness_policy.maximum_lag_delta if effective_data_time is None or effective_data_time < required_data_time: - return pendulum.Period(start=required_data_time, end=tick) + return PendulumInterval(start=required_data_time, end=tick) else: # occurs when asset is missing if effective_data_time is None: - return pendulum.Period( + return PendulumInterval( # require data from at most maximum_lag_delta ago start=current_time - freshness_policy.maximum_lag_delta, # this data should be available as soon as possible end=current_time, ) - return pendulum.Period( + return PendulumInterval( # we don't want to execute this too frequently start=effective_data_time + 0.9 * freshness_policy.maximum_lag_delta, end=max(effective_data_time + freshness_policy.maximum_lag_delta, current_time), @@ -64,7 +65,7 @@ def get_execution_period_and_evaluation_data_for_policies( policies: AbstractSet[FreshnessPolicy], effective_data_time: Optional[datetime.datetime], current_time: datetime.datetime, -) -> Tuple[Optional[pendulum.Period], Optional["TextRuleEvaluationData"]]: +) -> Tuple[Optional[PendulumInterval], Optional["TextRuleEvaluationData"]]: """Determines a range of times for which you can kick off an execution of this asset to solve the most pressing constraint, alongside a maximum number of additional constraints. """ @@ -84,7 +85,7 @@ def get_execution_period_and_evaluation_data_for_policies( if merged_period is None: merged_period = period elif period.start <= merged_period.end: - merged_period = pendulum.Period( + merged_period = PendulumInterval( start=max(period.start, merged_period.start), end=period.end, ) diff --git a/python_modules/dagster/dagster/_core/definitions/freshness_policy.py b/python_modules/dagster/dagster/_core/definitions/freshness_policy.py index a00223b6130f7..ca7c2a40f5627 100644 --- a/python_modules/dagster/dagster/_core/definitions/freshness_policy.py +++ b/python_modules/dagster/dagster/_core/definitions/freshness_policy.py @@ -1,12 +1,11 @@ import datetime from typing import AbstractSet, NamedTuple, Optional -import pendulum - import dagster._check as check from dagster._annotations import experimental from dagster._core.errors import DagsterInvalidDefinitionError from dagster._serdes import whitelist_for_serdes +from dagster._seven.compat.pendulum import pendulum_create_timezone from dagster._utils.schedules import ( is_valid_cron_schedule, reverse_cron_string_iterator, @@ -122,7 +121,7 @@ def __new__( ) try: # Verify that the timezone can be loaded - pendulum.tz.timezone(cron_schedule_timezone) # type: ignore + pendulum_create_timezone(cron_schedule_timezone) # type: ignore except Exception as e: raise DagsterInvalidDefinitionError( "Invalid cron schedule timezone '{cron_schedule_timezone}'. " diff --git a/python_modules/dagster/dagster/_core/definitions/schedule_definition.py b/python_modules/dagster/dagster/_core/definitions/schedule_definition.py index 0b4a7f74b8861..2cd6b39bcad8c 100644 --- a/python_modules/dagster/dagster/_core/definitions/schedule_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/schedule_definition.py @@ -20,7 +20,6 @@ cast, ) -import pendulum from typing_extensions import TypeAlias import dagster._check as check @@ -29,6 +28,7 @@ from dagster._core.definitions.resource_annotation import get_resource_args from dagster._core.definitions.scoped_resources_builder import Resources, ScopedResourcesBuilder from dagster._serdes import whitelist_for_serdes +from dagster._seven.compat.pendulum import pendulum_create_timezone from dagster._utils import IHasInternalInit, ensure_gen from dagster._utils.merger import merge_dicts from dagster._utils.schedules import is_valid_cron_schedule @@ -692,7 +692,7 @@ def _execution_fn(context: ScheduleEvaluationContext) -> RunRequestIterator: if self._execution_timezone: try: # Verify that the timezone can be loaded - pendulum.tz.timezone(self._execution_timezone) # type: ignore + pendulum_create_timezone(self._execution_timezone) except Exception as e: raise DagsterInvalidDefinitionError( f"Invalid execution timezone {self._execution_timezone} for {name}" diff --git a/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py b/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py index ae91daccf5a10..a4fb3ab38a5e8 100644 --- a/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py +++ b/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py @@ -42,7 +42,8 @@ unpack_value, ) from dagster._seven.compat.pendulum import ( - _IS_PENDULUM_2, + _IS_PENDULUM_1, + PRE_TRANSITION, PendulumDateTime, create_pendulum_time, to_timezone, @@ -146,7 +147,7 @@ def dst_safe_strptime(date_string: str, tz: str, fmt: str) -> PendulumDateTime: # Pendulum 1.x erroneously believes that there are two instances of the *second* hour after # a datetime transition, so to work around this we calculate the timestamp of the next # microsecond of the given datetime. - dt_microsecond = dt.microsecond + 1 if not _IS_PENDULUM_2 else dt.microsecond + dt_microsecond = dt.microsecond + 1 if _IS_PENDULUM_1 else dt.microsecond dt = create_pendulum_time( dt.year, dt.month, @@ -156,9 +157,9 @@ def dst_safe_strptime(date_string: str, tz: str, fmt: str) -> PendulumDateTime: dt.second, dt_microsecond, tz=tz, - dst_rule=pendulum.PRE_TRANSITION, + dst_rule=PRE_TRANSITION, ) - if not _IS_PENDULUM_2: + if _IS_PENDULUM_1: dt = dt.add(microseconds=-1) return dt diff --git a/python_modules/dagster/dagster/_seven/compat/pendulum.py b/python_modules/dagster/dagster/_seven/compat/pendulum.py index d705dd25ad7cb..9c212324f4db6 100644 --- a/python_modules/dagster/dagster/_seven/compat/pendulum.py +++ b/python_modules/dagster/dagster/_seven/compat/pendulum.py @@ -5,24 +5,42 @@ import pendulum from typing_extensions import TypeAlias -_IS_PENDULUM_2 = ( +_IS_PENDULUM_1 = ( hasattr(pendulum, "__version__") - and getattr(packaging.version.parse(getattr(pendulum, "__version__")), "major") == 2 + and getattr(packaging.version.parse(getattr(pendulum, "__version__")), "major") == 1 ) +_IS_PENDULUM_3 = ( + hasattr(pendulum, "__version__") + and getattr(packaging.version.parse(getattr(pendulum, "__version__")), "major") == 3 +) + +POST_TRANSITION = pendulum.tz.POST_TRANSITION if _IS_PENDULUM_3 else pendulum.POST_TRANSITION +PRE_TRANSITION = pendulum.tz.PRE_TRANSITION if _IS_PENDULUM_3 else pendulum.PRE_TRANSITION +TRANSITION_ERROR = pendulum.tz.TRANSITION_ERROR if _IS_PENDULUM_3 else pendulum.TRANSITION_ERROR + + +def pendulum_create_timezone(tz_name: str): + if _IS_PENDULUM_3: + from pendulum.tz.timezone import Timezone + + return Timezone(tz_name) + else: + return pendulum.tz.timezone(tz_name) # type: ignore + @contextmanager def mock_pendulum_timezone(override_timezone): - if _IS_PENDULUM_2: - with pendulum.tz.test_local_timezone(pendulum.tz.timezone(override_timezone)): + if _IS_PENDULUM_1: + with pendulum.tz.LocalTimezone.test(pendulum.Timezone.load(override_timezone)): yield else: - with pendulum.tz.LocalTimezone.test(pendulum.Timezone.load(override_timezone)): + with pendulum.tz.test_local_timezone(pendulum.tz.timezone(override_timezone)): yield def create_pendulum_time(year, month, day, *args, **kwargs): - if "tz" in kwargs and "dst_rule" in kwargs and not _IS_PENDULUM_2: + if "tz" in kwargs and "dst_rule" in kwargs and _IS_PENDULUM_1: tz = pendulum.timezone(kwargs.pop("tz")) dst_rule = kwargs.pop("dst_rule") @@ -39,18 +57,53 @@ def create_pendulum_time(year, month, day, *args, **kwargs): ) ) + if "dst_rule" in kwargs and _IS_PENDULUM_3: + dst_rule = kwargs.pop("dst_rule") + if dst_rule == PRE_TRANSITION: + kwargs["fold"] = 0 + elif dst_rule == POST_TRANSITION: + kwargs["fold"] = 1 + elif dst_rule == TRANSITION_ERROR: + tz_name = kwargs.pop("tz") + assert tz_name + return pendulum.instance( + pendulum_create_timezone(tz_name).convert( + datetime.datetime( + year, + month, + day, + *args, + **kwargs, + ), + raise_on_unknown_times=True, + ) + ) + return ( - pendulum.datetime(year, month, day, *args, **kwargs) - if _IS_PENDULUM_2 - else pendulum.create(year, month, day, *args, **kwargs) + pendulum.create(year, month, day, *args, **kwargs) + if _IS_PENDULUM_1 + else pendulum.datetime(year, month, day, *args, **kwargs) ) PendulumDateTime: TypeAlias = ( - pendulum.DateTime if _IS_PENDULUM_2 else pendulum.Pendulum # type: ignore[attr-defined] + pendulum.Pendulum if _IS_PENDULUM_1 else pendulum.DateTime # type: ignore[attr-defined] +) + +PendulumInterval: TypeAlias = ( + pendulum.Interval if _IS_PENDULUM_3 else pendulum.Period # type: ignore[attr-defined] ) +@contextmanager +def pendulum_freeze_time(t): + if _IS_PENDULUM_3: + yield from pendulum.travel_to(t, freeze=True) + else: + with pendulum.test(t) as frozen_time: + yield frozen_time + + # Workaround for issue with .in_tz() in pendulum: # https://github.com/sdispater/pendulum/issues/535 def to_timezone(dt: PendulumDateTime, tz: str): diff --git a/python_modules/dagster/dagster/_utils/schedules.py b/python_modules/dagster/dagster/_utils/schedules.py index 304c28e46d90c..c034f2bbbe002 100644 --- a/python_modules/dagster/dagster/_utils/schedules.py +++ b/python_modules/dagster/dagster/_utils/schedules.py @@ -9,7 +9,13 @@ import dagster._check as check from dagster._core.definitions.partition import ScheduleType -from dagster._seven.compat.pendulum import PendulumDateTime, create_pendulum_time +from dagster._seven.compat.pendulum import ( + POST_TRANSITION, + PRE_TRANSITION, + TRANSITION_ERROR, + PendulumDateTime, + create_pendulum_time, +) # Monthly schedules with 29-31 won't reliably run every month MAX_DAY_OF_MONTH_WITH_GUARANTEED_MONTHLY_INTERVAL = 28 @@ -99,7 +105,7 @@ def _replace_date_fields( 0, 0, tz=pendulum_date.timezone_name, - dst_rule=pendulum.TRANSITION_ERROR, + dst_rule=TRANSITION_ERROR, ) except pendulum.tz.exceptions.NonExistingTime: # type: ignore # If we fall on a non-existant time (e.g. between 2 and 3AM during a DST transition) @@ -114,7 +120,7 @@ def _replace_date_fields( 0, 0, tz=pendulum_date.timezone_name, - dst_rule=pendulum.TRANSITION_ERROR, + dst_rule=TRANSITION_ERROR, ) except pendulum.tz.exceptions.AmbiguousTime: # type: ignore # For consistency, always choose the latter of the two possible times during a fall DST @@ -129,7 +135,7 @@ def _replace_date_fields( 0, 0, tz=pendulum_date.timezone_name, - dst_rule=pendulum.POST_TRANSITION, + dst_rule=POST_TRANSITION, ) return new_time @@ -436,7 +442,7 @@ def _get_dates_to_consider_after_ambigious_time( next_date.second, next_date.microsecond, tz=timezone_str, - dst_rule=pendulum.POST_TRANSITION, + dst_rule=POST_TRANSITION, ) dates_to_consider = [post_transition_time] @@ -458,7 +464,7 @@ def _get_dates_to_consider_after_ambigious_time( next_date.second, next_date.microsecond, tz=timezone_str, - dst_rule=pendulum.PRE_TRANSITION, + dst_rule=PRE_TRANSITION, ) dates_to_consider.append(pre_transition_time) @@ -493,7 +499,7 @@ def _get_dates_to_consider_after_ambigious_time( next_date.second, next_date.microsecond, tz=timezone_str, - dst_rule=pendulum.PRE_TRANSITION, + dst_rule=PRE_TRANSITION, ) dates_to_consider.append(curr_pre_transition_time) @@ -506,7 +512,7 @@ def _get_dates_to_consider_after_ambigious_time( next_date.second, next_date.microsecond, tz=timezone_str, - dst_rule=pendulum.POST_TRANSITION, + dst_rule=POST_TRANSITION, ) dates_to_consider.append(curr_post_transition_time) @@ -590,7 +596,7 @@ def _timezone_aware_cron_iter( next_date.second, next_date.microsecond, tz=timezone_str, - dst_rule=pendulum.TRANSITION_ERROR, + dst_rule=TRANSITION_ERROR, ) ] except pendulum.tz.exceptions.NonExistingTime: # type:ignore @@ -610,7 +616,7 @@ def _timezone_aware_cron_iter( 0, 0, tz=timezone_str, - dst_rule=pendulum.TRANSITION_ERROR, + dst_rule=TRANSITION_ERROR, ) ] except pendulum.tz.exceptions.AmbiguousTime: # type: ignore diff --git a/python_modules/dagster/dagster/_utils/test/schedule_storage.py b/python_modules/dagster/dagster/_utils/test/schedule_storage.py index 2f574565027d0..d8424506c7c42 100644 --- a/python_modules/dagster/dagster/_utils/test/schedule_storage.py +++ b/python_modules/dagster/dagster/_utils/test/schedule_storage.py @@ -27,6 +27,7 @@ ) from dagster._core.types.loadable_target_origin import LoadableTargetOrigin from dagster._seven import get_current_datetime_in_utc +from dagster._seven.compat.pendulum import pendulum_freeze_time from dagster._utils.error import SerializableErrorInfo @@ -327,7 +328,7 @@ def test_update_tick_to_success(self, storage): freeze_datetime = pendulum.now("UTC") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): updated_tick = tick.with_status(TickStatus.SUCCESS).with_run_info(run_id="1234") assert updated_tick.status == TickStatus.SUCCESS assert updated_tick.end_timestamp == freeze_datetime.timestamp() @@ -352,7 +353,7 @@ def test_update_tick_to_skip(self, storage): freeze_datetime = pendulum.now("UTC") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): updated_tick = tick.with_status(TickStatus.SKIPPED) assert updated_tick.status == TickStatus.SKIPPED assert updated_tick.end_timestamp == freeze_datetime.timestamp() @@ -376,7 +377,7 @@ def test_update_tick_to_failure(self, storage): freeze_datetime = pendulum.now("UTC") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): updated_tick = tick.with_status( TickStatus.FAILURE, error=SerializableErrorInfo(message="Error", stack=[], cls_name="TestError"), diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py index cc513459ebecd..b53d1a4768656 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py @@ -42,7 +42,7 @@ ) from dagster._core.instance import DynamicPartitionsStore from dagster._core.test_utils import instance_for_test -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time def to_external_asset_graph(assets, asset_checks=None) -> AssetGraph: @@ -195,7 +195,7 @@ def child(parent): asset_graph = asset_graph_from_assets([parent, child]) - with pendulum.test(create_pendulum_time(year=2022, month=1, day=3, hour=4)): + with pendulum_freeze_time(create_pendulum_time(year=2022, month=1, day=3, hour=4)): with instance_for_test() as instance: current_time = pendulum.now("UTC") diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py index ac81b56b54a45..1aca27d7b363b 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitioned_assets.py @@ -39,7 +39,7 @@ ASSET_PARTITION_RANGE_START_TAG, ) from dagster._core.test_utils import assert_namedtuple_lists_equal -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time @pytest.fixture(autouse=True) @@ -725,7 +725,7 @@ def upstream_asset(context): def downstream_asset(context, upstream_asset): return upstream_asset + 1 - with pendulum.test(create_pendulum_time(2020, 1, 2, 10, 0)): + with pendulum_freeze_time(create_pendulum_time(2020, 1, 2, 10, 0)): with pytest.raises( DagsterInvariantViolationError, match="invalid partition keys", diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitions_subset.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitions_subset.py index 72dfd69e7afa9..8d084f3543a84 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitions_subset.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_partitions_subset.py @@ -16,7 +16,7 @@ ) from dagster._core.errors import DagsterInvalidDeserializationVersionError from dagster._serdes import deserialize_value, serialize_value -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time def test_default_subset_cannot_deserialize_invalid_version(): @@ -165,7 +165,7 @@ def test_all_partitions_subset_static_partitions_def() -> None: def test_all_partitions_subset_time_window_partitions_def() -> None: - with pendulum.test(create_pendulum_time(2020, 1, 6, hour=10)): + with pendulum_freeze_time(create_pendulum_time(2020, 1, 6, hour=10)): time_window_partitions_def = DailyPartitionsDefinition(start_date="2020-01-01") all_subset = AllPartitionsSubset(time_window_partitions_def, Mock(), pendulum.now("UTC")) assert len(all_subset) == 5 diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py index cb3529057fcbf..a312869786e3f 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill.py @@ -65,7 +65,7 @@ ) from dagster._core.test_utils import environ, instance_for_test from dagster._serdes import deserialize_value, serialize_value -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time from dagster._utils import Counter, traced_counter from dagster._utils.caching_instance_queryer import CachingInstanceQueryer @@ -287,7 +287,7 @@ def test_scenario_to_completion(scenario: AssetBackfillScenario, failures: str, ): instance.add_dynamic_partitions("foo", ["a", "b"]) - with pendulum.test(scenario.evaluation_time): + with pendulum_freeze_time(scenario.evaluation_time): assets_by_repo_name = scenario.assets_by_repo_name asset_graph = get_asset_graph(assets_by_repo_name) diff --git a/python_modules/dagster/dagster_tests/core_tests/test_data_time.py b/python_modules/dagster/dagster_tests/core_tests/test_data_time.py index 9b58e8dd192ee..b7c20b0c7c525 100644 --- a/python_modules/dagster/dagster_tests/core_tests/test_data_time.py +++ b/python_modules/dagster/dagster_tests/core_tests/test_data_time.py @@ -27,7 +27,7 @@ from dagster._core.definitions.observe import observe from dagster._core.definitions.time_window_partitions import DailyPartitionsDefinition from dagster._core.event_api import EventRecordsFilter -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time from dagster._utils.caching_instance_queryer import CachingInstanceQueryer @@ -276,7 +276,9 @@ class PartitionedDataTimeScenario(NamedTuple): @pytest.mark.parametrize("scenario", list(scenarios.values()), ids=list(scenarios.keys())) def test_partitioned_data_time(scenario): - with DagsterInstance.ephemeral() as instance, pendulum.test(create_pendulum_time(2023, 1, 7)): + with DagsterInstance.ephemeral() as instance, pendulum_freeze_time( + create_pendulum_time(2023, 1, 7) + ): _materialize_partitions(instance, scenario.before_partitions) record = _get_record(instance=instance) _materialize_partitions(instance, scenario.after_partitions) diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_asset_sensor_run.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_asset_sensor_run.py index 49dee0d7cd623..a4586ed73d083 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_asset_sensor_run.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_asset_sensor_run.py @@ -1,7 +1,6 @@ -import pendulum from dagster import materialize from dagster._core.scheduler.instigation import TickStatus -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from .test_run_status_sensors import ( instance_with_single_code_location_multiple_repos_with_sensors, @@ -25,7 +24,7 @@ def test_monitor_source_asset_sensor(executor): repos, ): asset_sensor_repo = repos["asset_sensor_repo"] - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): the_sensor = asset_sensor_repo.get_external_sensor("monitor_source_asset_sensor") instance.start_sensor(the_sensor) @@ -41,7 +40,7 @@ def test_monitor_source_asset_sensor(executor): ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): materialize([a_source_asset], instance=instance) evaluate_sensors(workspace_ctx, executor) diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py index 43faf4b447568..6f3af1fa63746 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_pythonic_resources.py @@ -3,7 +3,6 @@ from contextlib import contextmanager from typing import Iterator, Optional -import pendulum import pytest from dagster import ( AssetKey, @@ -51,7 +50,7 @@ from dagster._core.types.loadable_target_origin import LoadableTargetOrigin from dagster._core.workspace.context import WorkspaceProcessContext from dagster._core.workspace.load_target import ModuleTarget -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from .test_sensor_run import evaluate_sensors, validate_tick, wait_for_all_runs_to_start @@ -430,7 +429,7 @@ def test_resources( "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): base_run_count = 0 if "asset" in sensor_name: the_job.execute_in_process(instance=instance) @@ -499,7 +498,7 @@ def test_resources_freshness_policy_sensor( ) original_time = freeze_datetime - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo_struct_resources.get_external_sensor(sensor_name) instance.add_instigator_state( InstigatorState( @@ -515,15 +514,15 @@ def test_resources_freshness_policy_sensor( # We have to do two ticks because the first tick will be skipped due to the freshness policy # sensor initializing its cursor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context_struct_resources, None) wait_for_all_runs_to_start(instance) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context_struct_resources, None) wait_for_all_runs_to_start(instance) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): ticks = instance.get_ticks( external_sensor.get_external_origin_id(), external_sensor.selector_id ) @@ -575,7 +574,7 @@ def test_resources_run_status_sensor( ) original_time = freeze_datetime - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo_struct_resources.get_external_sensor(sensor_name) instance.add_instigator_state( InstigatorState( @@ -591,16 +590,16 @@ def test_resources_run_status_sensor( # We have to do two ticks because the first tick will be skipped due to the run status # sensor initializing its cursor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context_struct_resources, None) wait_for_all_runs_to_start(instance) the_job.execute_in_process(instance=instance) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context_struct_resources, None) wait_for_all_runs_to_start(instance) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): ticks = instance.get_ticks( external_sensor.get_external_origin_id(), external_sensor.selector_id ) @@ -657,7 +656,7 @@ def test_resources_run_failure_sensor( ) original_time = freeze_datetime - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo_struct_resources.get_external_sensor(sensor_name) instance.add_instigator_state( InstigatorState( @@ -673,16 +672,16 @@ def test_resources_run_failure_sensor( # We have to do two ticks because the first tick will be skipped due to the run status # sensor initializing its cursor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context_struct_resources, None) wait_for_all_runs_to_start(instance) the_failure_job.execute_in_process(instance=instance, raise_on_error=False) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context_struct_resources, None) wait_for_all_runs_to_start(instance) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): ticks = instance.get_ticks( external_sensor.get_external_origin_id(), external_sensor.selector_id ) diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_run_status_sensors.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_run_status_sensors.py index 7dad66a952936..57a4961961a68 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_run_status_sensors.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_run_status_sensors.py @@ -25,6 +25,7 @@ from dagster._core.test_utils import create_test_daemon_workspace_context, instance_for_test from dagster._core.workspace.context import WorkspaceProcessContext from dagster._core.workspace.load_target import WorkspaceFileTarget, WorkspaceLoadTarget +from dagster._seven.compat.pendulum import pendulum_freeze_time from .conftest import create_workspace_load_target from .test_sensor_run import ( @@ -129,7 +130,7 @@ def test_run_status_sensor( external_repo: ExternalRepository, ): freeze_datetime = pendulum.now() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): success_sensor = external_repo.get_external_sensor("my_job_success_sensor") instance.start_sensor(success_sensor) @@ -160,7 +161,7 @@ def test_run_status_sensor( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("failure_job") run = instance.create_run_for_job( failure_job, @@ -173,7 +174,7 @@ def test_run_status_sensor( assert run.status == DagsterRunStatus.FAILURE freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should not fire the success sensor, should fire the started sensro evaluate_sensors(workspace_context, executor) @@ -199,7 +200,7 @@ def test_run_status_sensor( TickStatus.SUCCESS, ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("foo_job") run = instance.create_run_for_job( foo_job, @@ -214,7 +215,7 @@ def test_run_status_sensor( caplog.clear() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire the success sensor and the started sensor evaluate_sensors(workspace_context, executor) @@ -251,7 +252,7 @@ def test_run_failure_sensor( external_repo: ExternalRepository, ): freeze_datetime = pendulum.now() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): failure_sensor = external_repo.get_external_sensor("my_run_failure_sensor") instance.start_sensor(failure_sensor) @@ -271,7 +272,7 @@ def test_run_failure_sensor( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("failure_job") run = instance.create_run_for_job( failure_job, @@ -284,7 +285,7 @@ def test_run_failure_sensor( assert run.status == DagsterRunStatus.FAILURE freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire the failure sensor evaluate_sensors(workspace_context, executor) @@ -307,7 +308,7 @@ def test_run_failure_sensor_that_fails( external_repo: ExternalRepository, ): freeze_datetime = pendulum.now() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): failure_sensor = external_repo.get_external_sensor( "my_run_failure_sensor_that_itself_fails" ) @@ -329,7 +330,7 @@ def test_run_failure_sensor_that_fails( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("failure_job") run = instance.create_run_for_job( failure_job, @@ -342,7 +343,7 @@ def test_run_failure_sensor_that_fails( assert run.status == DagsterRunStatus.FAILURE freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire the failure sensor and fail evaluate_sensors(workspace_context, executor) @@ -360,7 +361,7 @@ def test_run_failure_sensor_that_fails( # Next tick skips again freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire the failure sensor and fail evaluate_sensors(workspace_context, executor) @@ -383,7 +384,7 @@ def test_run_failure_sensor_filtered( external_repo: ExternalRepository, ): freeze_datetime = pendulum.now() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): failure_sensor = external_repo.get_external_sensor("my_run_failure_sensor_filtered") instance.start_sensor(failure_sensor) @@ -403,7 +404,7 @@ def test_run_failure_sensor_filtered( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("failure_job_2") run = instance.create_run_for_job( failure_job_2, @@ -416,7 +417,7 @@ def test_run_failure_sensor_filtered( assert run.status == DagsterRunStatus.FAILURE freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should not fire the failure sensor (filtered to failure job) evaluate_sensors(workspace_context, executor) @@ -434,7 +435,7 @@ def test_run_failure_sensor_filtered( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("failure_job") run = instance.create_run_for_job( failure_job, @@ -448,7 +449,7 @@ def test_run_failure_sensor_filtered( freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should not fire the failure sensor (filtered to failure job) evaluate_sensors(workspace_context, executor) @@ -508,7 +509,7 @@ def test_run_status_sensor_interleave(storage_config_fn, executor: Optional[Thre external_repo, ): # start sensor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): failure_sensor = external_repo.get_external_sensor("my_run_failure_sensor") instance.start_sensor(failure_sensor) @@ -528,7 +529,7 @@ def test_run_status_sensor_interleave(storage_config_fn, executor: Optional[Thre freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("hanging_job") # start run 1 run1 = instance.create_run_for_job( @@ -554,7 +555,7 @@ def test_run_status_sensor_interleave(storage_config_fn, executor: Optional[Thre assert run.run_id == run2.run_id # check sensor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire for run 2 evaluate_sensors(workspace_context, executor) @@ -572,14 +573,14 @@ def test_run_status_sensor_interleave(storage_config_fn, executor: Optional[Thre assert ticks[0].origin_run_ids[0] == run2.run_id # fail run 1 - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # fail run 2 instance.report_run_failed(run1) freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) # check sensor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire for run 1 evaluate_sensors(workspace_context, executor) @@ -608,7 +609,7 @@ def test_run_failure_sensor_empty_run_records( workspace_context, external_repo, ): - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): failure_sensor = external_repo.get_external_sensor("my_run_failure_sensor") instance.start_sensor(failure_sensor) @@ -628,7 +629,7 @@ def test_run_failure_sensor_empty_run_records( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # create a mismatch between event storage and run storage instance.event_log_storage.store_event( EventLogEntry( @@ -651,7 +652,7 @@ def test_run_failure_sensor_empty_run_records( assert len(failure_events) == 1 freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # shouldn't fire the failure sensor due to the mismatch evaluate_sensors(workspace_context, executor) @@ -703,7 +704,7 @@ def test_cross_code_location_run_status_sensor(executor: Optional[ThreadPoolExec workspace_context = daemon_sensor_defs_location_info.context # This remainder is largely copied from test_cross_repo_run_status_sensor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): success_sensor = sensor_repo.get_external_sensor("success_sensor") instance.start_sensor(success_sensor) @@ -725,7 +726,7 @@ def test_cross_code_location_run_status_sensor(executor: Optional[ThreadPoolExec freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_success_job = job_repo.get_full_external_job("success_job") # this unfortunate API (create_run_for_job) requires the importation @@ -745,7 +746,7 @@ def test_cross_code_location_run_status_sensor(executor: Optional[ThreadPoolExec assert dagster_run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = [ @@ -800,7 +801,7 @@ def test_cross_code_location_job_selector_on_defs_run_status_sensor( workspace_context = daemon_sensor_defs_location_info.context # This remainder is largely copied from test_cross_repo_run_status_sensor - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): success_sensor = sensor_repo.get_external_sensor("success_of_another_job_sensor") instance.start_sensor(success_sensor) @@ -822,7 +823,7 @@ def test_cross_code_location_job_selector_on_defs_run_status_sensor( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_success_job = job_repo.get_full_external_job("success_job") # this unfortunate API (create_run_for_job) requires the importation @@ -842,7 +843,7 @@ def test_cross_code_location_job_selector_on_defs_run_status_sensor( assert dagster_run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = [ @@ -869,7 +870,7 @@ def test_cross_code_location_job_selector_on_defs_run_status_sensor( # now launch the run that is actually being listened to - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_another_success_job = job_repo.get_full_external_job("another_success_job") # this unfortunate API (create_run_for_job) requires the importation @@ -889,7 +890,7 @@ def test_cross_code_location_job_selector_on_defs_run_status_sensor( assert dagster_run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = [ @@ -921,7 +922,7 @@ def test_cross_repo_run_status_sensor(executor: Optional[ThreadPoolExecutor]): the_repo = repos["the_repo"] the_other_repo = repos["the_other_repo"] - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): cross_repo_sensor = the_repo.get_external_sensor("cross_repo_sensor") instance.start_sensor(cross_repo_sensor) @@ -941,7 +942,7 @@ def test_cross_repo_run_status_sensor(executor: Optional[ThreadPoolExecutor]): freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = the_other_repo.get_full_external_job("the_job") run = instance.create_run_for_job( the_job, @@ -954,7 +955,7 @@ def test_cross_repo_run_status_sensor(executor: Optional[ThreadPoolExecutor]): assert run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( @@ -979,7 +980,7 @@ def test_cross_repo_job_run_status_sensor(executor: Optional[ThreadPoolExecutor] the_repo = repos["the_repo"] the_other_repo = repos["the_other_repo"] - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): cross_repo_sensor = the_repo.get_external_sensor("cross_repo_job_sensor") instance.start_sensor(cross_repo_sensor) @@ -1003,7 +1004,7 @@ def test_cross_repo_job_run_status_sensor(executor: Optional[ThreadPoolExecutor] freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = the_other_repo.get_full_external_job("the_job") run = instance.create_run_for_job( the_job, @@ -1017,7 +1018,7 @@ def test_cross_repo_job_run_status_sensor(executor: Optional[ThreadPoolExecutor] assert run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) wait_for_all_runs_to_finish(instance) @@ -1037,7 +1038,7 @@ def test_cross_repo_job_run_status_sensor(executor: Optional[ThreadPoolExecutor] assert run_request_runs[0].status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # ensure that the success of the run launched by the sensor doesn't trigger the sensor evaluate_sensors(workspace_context, executor) wait_for_all_runs_to_finish(instance) @@ -1064,7 +1065,7 @@ def test_partitioned_job_run_status_sensor( external_repo: ExternalRepository, ): freeze_datetime = pendulum.now() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): success_sensor = external_repo.get_external_sensor("partitioned_pipeline_success_sensor") instance.start_sensor(success_sensor) @@ -1086,7 +1087,7 @@ def test_partitioned_job_run_status_sensor( freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("daily_partitioned_job") run = instance.create_run_for_job( daily_partitioned_job, @@ -1103,7 +1104,7 @@ def test_partitioned_job_run_status_sensor( caplog.clear() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire the success sensor evaluate_sensors(workspace_context, executor) @@ -1135,7 +1136,7 @@ def test_different_instance_run_status_sensor(executor: Optional[ThreadPoolExecu the_other_workspace_context, the_other_repo, ): - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): cross_repo_sensor = the_repo.get_external_sensor("cross_repo_sensor") instance.start_sensor(cross_repo_sensor) @@ -1155,7 +1156,7 @@ def test_different_instance_run_status_sensor(executor: Optional[ThreadPoolExecu freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = the_other_repo.get_full_external_job("the_job") run = the_other_instance.create_run_for_job( the_job, @@ -1170,7 +1171,7 @@ def test_different_instance_run_status_sensor(executor: Optional[ThreadPoolExecu assert run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( @@ -1196,7 +1197,7 @@ def test_instance_run_status_sensor(executor: Optional[ThreadPoolExecutor]): the_repo = repos["the_repo"] the_other_repo = repos["the_other_repo"] - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance_sensor = the_repo.get_external_sensor("instance_sensor") instance.start_sensor(instance_sensor) @@ -1216,7 +1217,7 @@ def test_instance_run_status_sensor(executor: Optional[ThreadPoolExecutor]): freeze_datetime = freeze_datetime.add(seconds=60) time.sleep(1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = the_other_repo.get_full_external_job("the_job") run = instance.create_run_for_job( the_job, @@ -1229,7 +1230,7 @@ def test_instance_run_status_sensor(executor: Optional[ThreadPoolExecutor]): assert run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( @@ -1251,7 +1252,7 @@ def test_logging_run_status_sensor( external_repo: ExternalRepository, ): freeze_datetime = pendulum.now() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): success_sensor = external_repo.get_external_sensor("logging_status_sensor") instance.start_sensor(success_sensor) @@ -1270,7 +1271,7 @@ def test_logging_run_status_sensor( freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_job = external_repo.get_full_external_job("foo_job") run = instance.create_run_for_job( foo_job, @@ -1283,7 +1284,7 @@ def test_logging_run_status_sensor( assert run.status == DagsterRunStatus.SUCCESS freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire the success sensor and the started sensor evaluate_sensors(workspace_context, executor) diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_failure_recovery.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_failure_recovery.py index e5d6489098785..3eba1b64bccd5 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_failure_recovery.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_failure_recovery.py @@ -1,6 +1,5 @@ import multiprocessing -import pendulum import pytest from dagster._core.definitions.run_request import InstigatorType from dagster._core.instance import DagsterInstance @@ -17,7 +16,7 @@ from dagster._daemon import get_default_daemon_logger from dagster._daemon.sensor import execute_sensor_iteration from dagster._seven import IS_WINDOWS -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from .test_sensor_run import create_workspace_load_target, wait_for_all_runs_to_start @@ -27,7 +26,7 @@ def _test_launch_sensor_runs_in_subprocess(instance_ref, execution_datetime, debug_crash_flags): with DagsterInstance.from_ref(instance_ref) as instance: try: - with pendulum.test(execution_datetime), create_test_daemon_workspace_context( + with pendulum_freeze_time(execution_datetime), create_test_daemon_workspace_context( workspace_load_target=create_workspace_load_target(), instance=instance, ) as workspace_context: @@ -60,7 +59,7 @@ def test_failure_before_run_created(crash_location, crash_signal, instance, exte "US/Central", ) - with pendulum.test(frozen_datetime): + with pendulum_freeze_time(frozen_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") instance.add_instigator_state( InstigatorState( @@ -135,7 +134,7 @@ def test_failure_after_run_created_before_run_launched( create_pendulum_time(year=2019, month=2, day=28, hour=0, minute=0, second=0, tz="UTC"), "US/Central", ) - with pendulum.test(frozen_datetime): + with pendulum_freeze_time(frozen_datetime): external_sensor = external_repo.get_external_sensor("run_key_sensor") instance.add_instigator_state( InstigatorState( @@ -208,7 +207,7 @@ def test_failure_after_run_launched(crash_location, crash_signal, instance, exte ), "US/Central", ) - with pendulum.test(frozen_datetime): + with pendulum_freeze_time(frozen_datetime): external_sensor = external_repo.get_external_sensor("run_key_sensor") instance.add_instigator_state( InstigatorState( diff --git a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py index 63a32a97b66eb..86815a150fe81 100644 --- a/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py +++ b/python_modules/dagster/dagster_tests/daemon_sensor_tests/test_sensor_run.py @@ -84,7 +84,7 @@ from dagster._core.workspace.context import WorkspaceProcessContext from dagster._daemon import get_default_daemon_logger from dagster._daemon.sensor import execute_sensor_iteration, execute_sensor_iteration_loop -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from .conftest import create_workspace_load_target @@ -1013,7 +1013,7 @@ def test_ignore_automation_policy_sensor(instance, workspace_context, external_r "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("my_automation_policy_sensor") assert external_sensor instance.add_instigator_state( @@ -1040,7 +1040,7 @@ def test_simple_sensor(instance, workspace_context, external_repo, executor): "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") instance.add_instigator_state( InstigatorState( @@ -1071,7 +1071,7 @@ def test_simple_sensor(instance, workspace_context, external_repo, executor): freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 1 @@ -1105,7 +1105,7 @@ def test_sensors_keyed_on_selector_not_origin( "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") existing_origin = external_sensor.get_external_origin() @@ -1154,7 +1154,7 @@ def test_bad_load_sensor_repository( "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") valid_origin = external_sensor.get_external_origin() @@ -1194,7 +1194,7 @@ def test_bad_load_sensor(caplog, executor, instance, workspace_context, external "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") valid_origin = external_sensor.get_external_origin() @@ -1227,7 +1227,7 @@ def test_error_sensor(caplog, executor, instance, workspace_context, external_re create_pendulum_time(year=2019, month=2, day=27, hour=23, minute=59, second=59, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("error_sensor") instance.add_instigator_state( InstigatorState( @@ -1290,7 +1290,7 @@ def test_wrong_config_sensor(caplog, executor, instance, workspace_context, exte ), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("wrong_config_sensor") instance.add_instigator_state( InstigatorState( @@ -1325,7 +1325,7 @@ def test_wrong_config_sensor(caplog, executor, instance, workspace_context, exte freeze_datetime = freeze_datetime.add(seconds=60) caplog.clear() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Error repeats on subsequent ticks evaluate_sensors(workspace_context, executor) @@ -1360,7 +1360,7 @@ def test_launch_failure(caplog, executor, workspace_context, external_repo): }, }, ) as instance: - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): exploding_workspace_context = workspace_context.copy_for_test_instance(instance) external_sensor = external_repo.get_external_sensor("always_on_sensor") instance.add_instigator_state( @@ -1411,7 +1411,7 @@ def test_launch_once(caplog, executor, instance, workspace_context, external_rep "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("run_key_sensor") instance.add_instigator_state( InstigatorState( @@ -1445,7 +1445,7 @@ def test_launch_once(caplog, executor, instance, workspace_context, external_rep # run again (after 30 seconds), to ensure that the run key maintains idempotence freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) assert instance.get_runs_count() == 1 ticks = instance.get_ticks( @@ -1478,7 +1478,7 @@ def test_launch_once(caplog, executor, instance, workspace_context, external_rep # Sensor loop still executes freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( external_sensor.get_external_origin_id(), external_sensor.selector_id @@ -1497,7 +1497,7 @@ def test_custom_interval_sensor(executor, instance, workspace_context, external_ freeze_datetime = to_timezone( create_pendulum_time(year=2019, month=2, day=28, tz="UTC"), "US/Central" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("custom_interval_sensor") instance.add_instigator_state( InstigatorState( @@ -1520,7 +1520,7 @@ def test_custom_interval_sensor(executor, instance, workspace_context, external_ freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( external_sensor.get_external_origin_id(), external_sensor.selector_id @@ -1530,7 +1530,7 @@ def test_custom_interval_sensor(executor, instance, workspace_context, external_ freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( external_sensor.get_external_origin_id(), external_sensor.selector_id @@ -1559,7 +1559,7 @@ def fake_sleep(s): shutdown_event = mock.MagicMock() shutdown_event.wait.side_effect = fake_sleep - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # 60 second custom interval external_sensor = external_repo.get_external_sensor("custom_interval_sensor") @@ -1610,7 +1610,7 @@ def test_sensor_start_stop(executor, instance, workspace_context, external_repo) create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("always_on_sensor") external_origin_id = external_sensor.get_external_origin_id() instance.start_sensor(external_sensor) @@ -1635,7 +1635,7 @@ def test_sensor_start_stop(executor, instance, workspace_context, external_repo) freeze_datetime = freeze_datetime.add(seconds=15) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) # no new ticks, no new runs, we are below the 30 second min interval assert instance.get_runs_count() == 1 @@ -1654,7 +1654,7 @@ def test_sensor_start_stop(executor, instance, workspace_context, external_repo) freeze_datetime = freeze_datetime.add(seconds=16) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) # should have new tick, new run, we are after the 30 second min interval assert instance.get_runs_count() == 2 @@ -1667,7 +1667,7 @@ def test_large_sensor(executor, instance, workspace_context, external_repo): create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("large_sensor") instance.start_sensor(external_sensor) evaluate_sensors(workspace_context, executor, timeout=300) @@ -1688,7 +1688,7 @@ def test_many_request_sensor(executor, submit_executor, instance, workspace_cont create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("many_request_sensor") instance.start_sensor(external_sensor) evaluate_sensors(workspace_context, executor, submit_executor=submit_executor) @@ -1709,7 +1709,7 @@ def test_cursor_sensor(executor, instance, workspace_context, external_repo): create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): skip_sensor = external_repo.get_external_sensor("skip_cursor_sensor") run_sensor = external_repo.get_external_sensor("run_cursor_sensor") instance.start_sensor(skip_sensor) @@ -1739,7 +1739,7 @@ def test_cursor_sensor(executor, instance, workspace_context, external_repo): assert run_ticks[0].cursor == "1" freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) skip_ticks = instance.get_ticks( @@ -1770,7 +1770,7 @@ def test_run_request_asset_selection_sensor(executor, instance, workspace_contex create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("run_request_asset_selection_sensor") external_origin_id = external_sensor.get_external_origin_id() instance.start_sensor(external_sensor) @@ -1810,7 +1810,7 @@ def test_run_request_stale_asset_selection_sensor_never_materialized( "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("run_request_stale_asset_sensor") instance.start_sensor(external_sensor) evaluate_sensors(workspace_context, executor) @@ -1829,7 +1829,7 @@ def test_run_request_stale_asset_selection_sensor_empty( materialize([a, b, c], instance=instance) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("run_request_stale_asset_sensor") instance.start_sensor(external_sensor) evaluate_sensors(workspace_context, executor) @@ -1847,7 +1847,7 @@ def test_run_request_stale_asset_selection_sensor_subset( materialize([a], instance=instance) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("run_request_stale_asset_sensor") instance.start_sensor(external_sensor) evaluate_sensors(workspace_context, executor) @@ -1861,7 +1861,7 @@ def test_targets_asset_selection_sensor(executor, instance, workspace_context, e create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("targets_asset_selection_sensor") external_origin_id = external_sensor.get_external_origin_id() instance.start_sensor(external_sensor) @@ -1909,7 +1909,7 @@ def test_partitioned_asset_selection_sensor(executor, instance, workspace_contex create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("partitioned_asset_selection_sensor") external_origin_id = external_sensor.get_external_origin_id() instance.start_sensor(external_sensor) @@ -1948,7 +1948,7 @@ def test_asset_sensor(executor, instance, workspace_context, external_repo): create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): foo_sensor = external_repo.get_external_sensor("asset_foo_sensor") instance.start_sensor(foo_sensor) @@ -1964,7 +1964,7 @@ def test_asset_sensor(executor, instance, workspace_context, external_repo): ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate the foo asset foo_job.execute_in_process(instance=instance) @@ -1989,7 +1989,7 @@ def test_asset_job_sensor(executor, instance, workspace_context, external_repo): create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): job_sensor = external_repo.get_external_sensor("asset_job_sensor") instance.start_sensor(job_sensor) @@ -2006,7 +2006,7 @@ def test_asset_job_sensor(executor, instance, workspace_context, external_repo): assert "No new materialization events" in ticks[0].tick_data.skip_reason freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate the foo asset foo_job.execute_in_process(instance=instance) @@ -2033,7 +2033,7 @@ def test_asset_sensor_not_triggered_on_observation( create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): foo_sensor = external_repo.get_external_sensor("asset_foo_sensor") instance.start_sensor(foo_sensor) @@ -2053,7 +2053,7 @@ def test_asset_sensor_not_triggered_on_observation( ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate the foo asset foo_job.execute_in_process(instance=instance) @@ -2078,7 +2078,7 @@ def test_multi_asset_sensor(executor, instance, workspace_context, external_repo create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): a_and_b_sensor = external_repo.get_external_sensor("asset_a_and_b_sensor") instance.start_sensor(a_and_b_sensor) @@ -2096,7 +2096,7 @@ def test_multi_asset_sensor(executor, instance, workspace_context, external_repo ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate asset_a materialize([asset_a], instance=instance) @@ -2116,7 +2116,7 @@ def test_multi_asset_sensor(executor, instance, workspace_context, external_repo freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate asset_b materialize([asset_b], instance=instance) @@ -2143,7 +2143,7 @@ def test_asset_selection_sensor(executor, instance, workspace_context, external_ create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): asset_selection_sensor = external_repo.get_external_sensor("asset_selection_sensor") instance.start_sensor(asset_selection_sensor) @@ -2168,7 +2168,7 @@ def test_multi_asset_sensor_targets_asset_selection( create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): multi_asset_sensor_targets_asset_selection = external_repo.get_external_sensor( "multi_asset_sensor_targets_asset_selection" ) @@ -2189,7 +2189,7 @@ def test_multi_asset_sensor_targets_asset_selection( ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate asset_a materialize([asset_a], instance=instance) @@ -2210,7 +2210,7 @@ def test_multi_asset_sensor_targets_asset_selection( freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate asset_b materialize([asset_b], instance=instance) @@ -2239,7 +2239,7 @@ def test_multi_asset_sensor_w_many_events(executor, instance, workspace_context, create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): backlog_sensor = external_repo.get_external_sensor("backlog_sensor") instance.start_sensor(backlog_sensor) @@ -2257,7 +2257,7 @@ def test_multi_asset_sensor_w_many_events(executor, instance, workspace_context, ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate asset_a materialize([asset_a], instance=instance) @@ -2276,7 +2276,7 @@ def test_multi_asset_sensor_w_many_events(executor, instance, workspace_context, freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate asset_a materialize([asset_a], instance=instance) @@ -2305,7 +2305,7 @@ def test_multi_asset_sensor_w_no_cursor_update( create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): cursor_sensor = external_repo.get_external_sensor("doesnt_update_cursor_sensor") instance.start_sensor(cursor_sensor) @@ -2323,7 +2323,7 @@ def test_multi_asset_sensor_w_no_cursor_update( ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should generate asset_a materialize([asset_a], instance=instance) @@ -2345,7 +2345,7 @@ def test_multi_asset_sensor_hourly_to_weekly(executor, instance, workspace_conte create_pendulum_time(year=2022, month=8, day=2, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): materialize([hourly_asset], instance=instance, partition_key="2022-08-01-00:00") cursor_sensor = external_repo.get_external_sensor("multi_asset_sensor_hourly_to_weekly") instance.start_sensor(cursor_sensor) @@ -2374,7 +2374,7 @@ def test_multi_asset_sensor_hourly_to_hourly(executor, instance, workspace_conte create_pendulum_time(year=2022, month=8, day=3, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): materialize([hourly_asset], instance=instance, partition_key="2022-08-02-00:00") cursor_sensor = external_repo.get_external_sensor("multi_asset_sensor_hourly_to_hourly") instance.start_sensor(cursor_sensor) @@ -2400,7 +2400,7 @@ def test_multi_asset_sensor_hourly_to_hourly(executor, instance, workspace_conte freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): cursor_sensor = external_repo.get_external_sensor("multi_asset_sensor_hourly_to_hourly") instance.start_sensor(cursor_sensor) @@ -2418,7 +2418,7 @@ def test_sensor_result_multi_asset_sensor(executor, instance, workspace_context, create_pendulum_time(year=2022, month=8, day=3, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): cursor_sensor = external_repo.get_external_sensor("sensor_result_multi_asset_sensor") instance.start_sensor(cursor_sensor) @@ -2443,7 +2443,7 @@ def test_cursor_update_sensor_result_multi_asset_sensor( create_pendulum_time(year=2022, month=8, day=3, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): cursor_sensor = external_repo.get_external_sensor("cursor_sensor_result_multi_asset_sensor") instance.start_sensor(cursor_sensor) @@ -2467,7 +2467,7 @@ def test_multi_job_sensor(executor, instance, workspace_context, external_repo): create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): job_sensor = external_repo.get_external_sensor("two_job_sensor") instance.start_sensor(job_sensor) @@ -2488,7 +2488,7 @@ def test_multi_job_sensor(executor, instance, workspace_context, external_repo): assert run.job_name == "the_job" freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # should fire the asset sensor evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks(job_sensor.get_external_origin_id(), job_sensor.selector_id) @@ -2511,7 +2511,7 @@ def test_bad_run_request_untargeted(executor, instance, workspace_context, exter create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): job_sensor = external_repo.get_external_sensor("bad_request_untargeted") instance.start_sensor(job_sensor) @@ -2536,7 +2536,7 @@ def test_bad_run_request_mismatch(executor, instance, workspace_context, externa create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): job_sensor = external_repo.get_external_sensor("bad_request_mismatch") instance.start_sensor(job_sensor) @@ -2560,7 +2560,7 @@ def test_bad_run_request_unspecified(executor, instance, workspace_context, exte create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): job_sensor = external_repo.get_external_sensor("bad_request_unspecified") instance.start_sensor(job_sensor) @@ -2593,7 +2593,7 @@ def test_status_in_code_sensor(executor, instance): iter(workspace_context.create_request_context().get_workspace_snapshot().values()) ).code_location.get_repository("the_status_in_code_repo") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): running_sensor = external_repo.get_external_sensor("always_running_sensor") not_running_sensor = external_repo.get_external_sensor("never_running_sensor") @@ -2683,7 +2683,7 @@ def test_status_in_code_sensor(executor, instance): evaluate_sensors(workspace_context, executor) freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 1 @@ -2721,7 +2721,7 @@ def test_run_request_list_sensor(executor, instance, workspace_context, external create_pendulum_time(year=2019, month=2, day=27, hour=23, minute=59, second=59, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("request_list_sensor") instance.add_instigator_state( InstigatorState( @@ -2750,7 +2750,7 @@ def test_sensor_purge(executor, instance, workspace_context, external_repo): create_pendulum_time(year=2019, month=2, day=27, hour=23, minute=59, second=59, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") instance.add_instigator_state( InstigatorState( @@ -2773,7 +2773,7 @@ def test_sensor_purge(executor, instance, workspace_context, external_repo): assert len(ticks) == 1 freeze_datetime = freeze_datetime.add(days=6) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # create another tick evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( @@ -2783,7 +2783,7 @@ def test_sensor_purge(executor, instance, workspace_context, external_repo): freeze_datetime = freeze_datetime.add(days=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # create another tick, but the first tick should be purged evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( @@ -2804,7 +2804,7 @@ def test_sensor_custom_purge(executor, workspace_context, external_repo): }, ) as instance: purge_ws_ctx = workspace_context.copy_for_test_instance(instance) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") instance.add_instigator_state( InstigatorState( @@ -2827,7 +2827,7 @@ def test_sensor_custom_purge(executor, workspace_context, external_repo): assert len(ticks) == 1 freeze_datetime = freeze_datetime.add(days=8) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # create another tick, and the first tick should not be purged despite the fact that the # default purge day offset is 7 evaluate_sensors(purge_ws_ctx, executor) @@ -2838,7 +2838,7 @@ def test_sensor_custom_purge(executor, workspace_context, external_repo): freeze_datetime = freeze_datetime.add(days=7) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # create another tick, but the first tick should be purged evaluate_sensors(purge_ws_ctx, executor) ticks = instance.get_ticks( @@ -2885,7 +2885,7 @@ def test_repository_namespacing(executor): external_sensor = external_repo.get_external_sensor("run_key_sensor") other_sensor = other_repo.get_external_sensor("run_key_sensor") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_sensor(external_sensor) assert instance.get_runs_count() == 0 ticks = instance.get_ticks( @@ -2919,7 +2919,7 @@ def test_repository_namespacing(executor): # run again (after 30 seconds), to ensure that the run key maintains idempotence freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(full_workspace_context, executor) assert instance.get_runs_count() == 2 # still 2 ticks = instance.get_ticks( @@ -3090,7 +3090,7 @@ def test_add_delete_skip_dynamic_partitions( "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( @@ -3124,7 +3124,7 @@ def test_add_delete_skip_dynamic_partitions( assert run.tags.get("dagster/partition") == "1" freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) ticks = instance.get_ticks( @@ -3297,7 +3297,7 @@ def test_stale_request_context(instance, workspace_context, external_repo): executor = ThreadPoolExecutor() blocking_executor = BlockingThreadPoolExecutor() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") instance.add_instigator_state( InstigatorState( @@ -3340,7 +3340,7 @@ def test_stale_request_context(instance, workspace_context, external_repo): blocking_executor.block() freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): futures = {} list( execute_sensor_iteration( @@ -3389,7 +3389,7 @@ def test_start_tick_sensor(executor, instance, workspace_context, external_repo) create_pendulum_time(year=2019, month=2, day=27, tz="UTC"), "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): start_skip_sensor = external_repo.get_external_sensor("start_skip_sensor") instance.start_sensor(start_skip_sensor) evaluate_sensors(workspace_context, executor) @@ -3401,7 +3401,7 @@ def test_start_tick_sensor(executor, instance, workspace_context, external_repo) ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) validate_tick( _get_last_tick(instance, start_skip_sensor), @@ -3411,7 +3411,7 @@ def test_start_tick_sensor(executor, instance, workspace_context, external_repo) ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.stop_sensor( start_skip_sensor.get_external_origin_id(), start_skip_sensor.selector_id, @@ -3427,7 +3427,7 @@ def test_start_tick_sensor(executor, instance, workspace_context, external_repo) ) freeze_datetime = freeze_datetime.add(seconds=60) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_sensors(workspace_context, executor) validate_tick( _get_last_tick(instance, start_skip_sensor), diff --git a/python_modules/dagster/dagster_tests/daemon_tests/test_backfill_failure_recovery.py b/python_modules/dagster/dagster_tests/daemon_tests/test_backfill_failure_recovery.py index 62ffe45fad5f1..85f46ec981da9 100644 --- a/python_modules/dagster/dagster_tests/daemon_tests/test_backfill_failure_recovery.py +++ b/python_modules/dagster/dagster_tests/daemon_tests/test_backfill_failure_recovery.py @@ -16,7 +16,7 @@ from dagster._daemon import get_default_daemon_logger from dagster._daemon.backfill import execute_backfill_iteration from dagster._seven import IS_WINDOWS -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from .conftest import workspace_load_target @@ -34,7 +34,7 @@ def _test_backfill_in_subprocess(instance_ref, debug_crash_flags): ) with DagsterInstance.from_ref(instance_ref) as instance: try: - with pendulum.test(execution_datetime), create_test_daemon_workspace_context( + with pendulum_freeze_time(execution_datetime), create_test_daemon_workspace_context( workspace_load_target=workspace_load_target(), instance=instance ) as workspace_context: list( diff --git a/python_modules/dagster/dagster_tests/daemon_tests/test_concurrency_daemon.py b/python_modules/dagster/dagster_tests/daemon_tests/test_concurrency_daemon.py index c9f99e15b75ce..4c9f1b6843853 100644 --- a/python_modules/dagster/dagster_tests/daemon_tests/test_concurrency_daemon.py +++ b/python_modules/dagster/dagster_tests/daemon_tests/test_concurrency_daemon.py @@ -1,7 +1,6 @@ import tempfile from logging import Logger -import pendulum import pytest from dagster._core.instance import DagsterInstance from dagster._core.storage.dagster_run import DagsterRunStatus @@ -14,7 +13,7 @@ from dagster._core.workspace.load_target import EmptyWorkspaceTarget from dagster._daemon import get_default_daemon_logger from dagster._daemon.monitoring.concurrency import execute_concurrency_slots_iteration -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time @pytest.fixture @@ -59,7 +58,7 @@ def test_global_concurrency_release( tz="UTC", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): run = create_run_for_test(instance, job_name="my_job", status=DagsterRunStatus.STARTING) instance.event_log_storage.claim_concurrency_slot("foo", run.run_id, "my_step") key_info = instance.event_log_storage.get_concurrency_info("foo") @@ -68,14 +67,14 @@ def test_global_concurrency_release( instance.report_run_canceled(run) freeze_datetime = freeze_datetime.add(seconds=59) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): list(execute_concurrency_slots_iteration(workspace_context, logger)) key_info = instance.event_log_storage.get_concurrency_info("foo") assert key_info.slot_count == 1 assert key_info.active_slot_count == 1 freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): list(execute_concurrency_slots_iteration(workspace_context, logger)) key_info = instance.event_log_storage.get_concurrency_info("foo") assert key_info.slot_count == 1 diff --git a/python_modules/dagster/dagster_tests/daemon_tests/test_monitoring_daemon.py b/python_modules/dagster/dagster_tests/daemon_tests/test_monitoring_daemon.py index e905be147abb0..b1c652db03b81 100644 --- a/python_modules/dagster/dagster_tests/daemon_tests/test_monitoring_daemon.py +++ b/python_modules/dagster/dagster_tests/daemon_tests/test_monitoring_daemon.py @@ -29,6 +29,7 @@ ) from dagster._serdes import ConfigurableClass from dagster._serdes.config_class import ConfigurableClassData +from dagster._seven.compat.pendulum import pendulum_freeze_time from typing_extensions import Self @@ -285,7 +286,7 @@ def test_long_running_termination( ): with environ({"DAGSTER_TEST_RUN_HEALTH_CHECK_RESULT": "healthy"}): initial = pendulum.datetime(2021, 1, 1, tz="UTC") - with pendulum.test(initial): + with pendulum_freeze_time(initial): too_long_run = create_run_for_test( instance, job_name="foo", @@ -302,7 +303,7 @@ def test_long_running_termination( instance, job_name="foo", status=DagsterRunStatus.STARTING ) started_time = initial.add(seconds=1) - with pendulum.test(started_time): + with pendulum_freeze_time(started_time): report_started_event(instance, too_long_run, started_time.timestamp()) report_started_event(instance, okay_run, started_time.timestamp()) report_started_event(instance, run_no_tag, started_time.timestamp()) @@ -326,7 +327,7 @@ def test_long_running_termination( run_launcher = cast(TestRunLauncher, instance.run_launcher) eval_time = started_time.add(seconds=501) - with pendulum.test(eval_time): + with pendulum_freeze_time(eval_time): # run_no_tag has no maximum run tag set, so no termination event should be # triggered. monitor_started_run(instance, workspace, no_tag_record, logger) @@ -373,7 +374,7 @@ def test_long_running_termination_failure( else: instance.run_launcher.should_except_termination = True # type: ignore initial = pendulum.datetime(2021, 1, 1, tz="UTC") - with pendulum.test(initial): + with pendulum_freeze_time(initial): too_long_run = create_run_for_test( instance, job_name="foo", @@ -381,7 +382,7 @@ def test_long_running_termination_failure( tags={MAX_RUNTIME_SECONDS_TAG: "500"}, ) started_time = initial.add(seconds=1) - with pendulum.test(started_time): + with pendulum_freeze_time(started_time): report_started_event(instance, too_long_run, started_time.timestamp()) too_long_record = instance.get_run_record_by_id(too_long_run.run_id) @@ -393,7 +394,7 @@ def test_long_running_termination_failure( run_launcher = cast(TestRunLauncher, instance.run_launcher) eval_time = started_time.add(seconds=501) - with pendulum.test(eval_time): + with pendulum_freeze_time(eval_time): # Enough runtime has elapsed for too_long_run to hit its maximum runtime so a # termination event should be triggered. monitor_started_run(instance, workspace, too_long_record, logger) diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py index 25834470495eb..c1007150073bc 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py @@ -92,6 +92,7 @@ get_current_evaluation_id, ) from dagster._serdes.serdes import serialize_value +from dagster._seven.compat.pendulum import pendulum_freeze_time from .base_scenario import FAIL_TAG, run_request @@ -348,7 +349,7 @@ def test_time_fn() -> float: # fake current_time on the scenario state return (self.current_time + (datetime.datetime.now() - start)).timestamp() - with pendulum.test(self.current_time), mock.patch("time.time", new=test_time_fn): + with pendulum_freeze_time(self.current_time), mock.patch("time.time", new=test_time_fn): for rr in run_requests: materialize( assets=self.assets, @@ -541,7 +542,8 @@ def evaluate_tick(self, label: Optional[str] = None) -> "AssetDaemonScenarioStat self.logger.critical("********************************") self.logger.critical(f"EVALUATING TICK {label or self.tick_index}") self.logger.critical("********************************") - with pendulum.test(self.current_time): + + with pendulum_freeze_time(self.current_time): if self.is_daemon: ( new_run_requests, diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/base_scenario.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/base_scenario.py index 7839b80cea46d..d74e6251c2560 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/base_scenario.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/base_scenario.py @@ -86,6 +86,7 @@ asset_daemon_cursor_from_pre_sensor_auto_materialize_serialized_cursor, ) from dagster._serdes.serdes import serialize_value +from dagster._seven.compat.pendulum import pendulum_freeze_time from dagster._utils import SingleInstigatorDebugCrashFlags @@ -262,7 +263,7 @@ def do_sensor_scenario( test_time = self.current_time or pendulum.now() - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): @repository def repo(): @@ -362,7 +363,7 @@ def test_time_fn(): if self.between_runs_delta is not None: test_time += self.between_runs_delta - with pendulum.test(test_time), mock.patch("time.time", new=test_time_fn): + with pendulum_freeze_time(test_time), mock.patch("time.time", new=test_time_fn): if run.is_observation: observe( instance=instance, @@ -383,7 +384,7 @@ def test_time_fn(): if self.evaluation_delta is not None: test_time += self.evaluation_delta - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): # get asset_graph if not with_external_asset_graph: asset_graph = repo.asset_graph @@ -446,7 +447,7 @@ def do_daemon_scenario( test_time = self.current_time or pendulum.now() - with pendulum.test(test_time) if self.current_time else contextlib.nullcontext(): + with pendulum_freeze_time(test_time) if self.current_time else contextlib.nullcontext(): if self.cursor_from is not None: self.cursor_from.do_daemon_scenario( instance, @@ -462,7 +463,7 @@ def test_time_fn(): if self.between_runs_delta is not None: test_time += self.between_runs_delta - with pendulum.test(test_time), mock.patch("time.time", new=test_time_fn): + with pendulum_freeze_time(test_time), mock.patch("time.time", new=test_time_fn): assert not run.is_observation, "Observations not supported for daemon tests" if self.assets: do_run( @@ -488,7 +489,7 @@ def test_time_fn(): if self.evaluation_delta is not None: test_time += self.evaluation_delta - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): assert scenario_name is not None, "scenario_name must be provided for daemon runs" if self.code_locations: diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_failure_recovery.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_failure_recovery.py index 82eecfbfa5fd4..7eee1909ba0cd 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_failure_recovery.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_failure_recovery.py @@ -1,7 +1,6 @@ import multiprocessing from typing import TYPE_CHECKING -import pendulum import pytest from dagster import AssetKey from dagster._core.errors import DagsterUserCodeUnreachableError @@ -22,6 +21,7 @@ _get_pre_sensor_auto_materialize_serialized_cursor, set_auto_materialize_paused, ) +from dagster._seven.compat.pendulum import pendulum_freeze_time from dagster._utils import SingleInstigatorDebugCrashFlags, get_terminate_signal from .scenarios.auto_materialize_policy_scenarios import auto_materialize_policy_scenarios @@ -91,7 +91,7 @@ def test_old_tick_not_resumed(daemon_not_paused_instance): debug_crash_flags = {"RUN_CREATED": Exception("OOPS")} - with pendulum.test(execution_time): + with pendulum_freeze_time(execution_time): with pytest.raises(Exception, match="OOPS"): error_asset_scenario.do_daemon_scenario( instance, @@ -110,7 +110,7 @@ def test_old_tick_not_resumed(daemon_not_paused_instance): # advancing past MAX_TIME_TO_RESUME_TICK_SECONDS gives up and advances to a new evaluation execution_time = execution_time.add(seconds=MAX_TIME_TO_RESUME_TICK_SECONDS + 1) - with pendulum.test(execution_time): + with pendulum_freeze_time(execution_time): with pytest.raises(Exception, match="OOPS"): error_asset_scenario.do_daemon_scenario( instance, @@ -128,7 +128,7 @@ def test_old_tick_not_resumed(daemon_not_paused_instance): # advancing less than that retries the same tick execution_time = execution_time.add(seconds=MAX_TIME_TO_RESUME_TICK_SECONDS - 1) - with pendulum.test(execution_time): + with pendulum_freeze_time(execution_time): with pytest.raises(Exception, match="OOPS"): error_asset_scenario.do_daemon_scenario( instance, @@ -162,7 +162,7 @@ def test_error_loop_before_cursor_written(daemon_not_paused_instance, crash_loca for trial_num in range(3): test_time = execution_time.add(seconds=15 * trial_num) - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): debug_crash_flags = {crash_location: Exception(f"Oops {trial_num}")} with pytest.raises(Exception, match=f"Oops {trial_num}"): @@ -200,7 +200,7 @@ def test_error_loop_before_cursor_written(daemon_not_paused_instance, crash_loca assert not cursor test_time = test_time.add(seconds=45) - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): # Next successful tick recovers error_asset_scenario.do_daemon_scenario( instance, @@ -245,7 +245,7 @@ def test_error_loop_after_cursor_written(daemon_not_paused_instance, crash_locat # User code error retries but does not increment the retry count test_time = execution_time.add(seconds=15) - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): debug_crash_flags = {crash_location: DagsterUserCodeUnreachableError("WHERE IS THE CODE")} with pytest.raises( @@ -289,7 +289,7 @@ def test_error_loop_after_cursor_written(daemon_not_paused_instance, crash_locat for trial_num in range(3): test_time = test_time.add(seconds=15) - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): debug_crash_flags = {crash_location: Exception(f"Oops {trial_num}")} with pytest.raises(Exception, match=f"Oops {trial_num}"): @@ -329,7 +329,7 @@ def test_error_loop_after_cursor_written(daemon_not_paused_instance, crash_locat # Next tick moves on to use the new cursor / evaluation ID since we have passed the maximum # number of retries test_time = test_time.add(seconds=45) - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): debug_crash_flags = {"RUN_IDS_ADDED_TO_EVALUATIONS": Exception("Oops new tick")} with pytest.raises(Exception, match="Oops new tick"): error_asset_scenario.do_daemon_scenario( @@ -358,7 +358,7 @@ def test_error_loop_after_cursor_written(daemon_not_paused_instance, crash_locat assert moved_on_cursor != last_cursor test_time = test_time.add(seconds=45) - with pendulum.test(test_time): + with pendulum_freeze_time(test_time): # Next successful tick recovers error_asset_scenario.do_daemon_scenario( instance, diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_partitioned_schedule.py b/python_modules/dagster/dagster_tests/definitions_tests/test_partitioned_schedule.py index e6a3801276109..10f6881ba3416 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_partitioned_schedule.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_partitioned_schedule.py @@ -27,6 +27,7 @@ monthly_partitioned_config, weekly_partitioned_config, ) +from dagster._seven.compat.pendulum import pendulum_freeze_time DATE_FORMAT = "%Y-%m-%d" @@ -390,7 +391,7 @@ def my_partitioned_config(start, end): def test_future_tick(): - with pendulum.test(pendulum.parse("2022-02-28")): + with pendulum_freeze_time(pendulum.parse("2022-02-28")): @daily_partitioned_config(start_date="2021-05-05") def my_partitioned_config(start, end): diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/test_cron_string_iterator.py b/python_modules/dagster/dagster_tests/scheduler_tests/test_cron_string_iterator.py index 80b4d1bc06a59..f533ff5e8fd87 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/test_cron_string_iterator.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/test_cron_string_iterator.py @@ -2,7 +2,12 @@ import pendulum import pytest -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import ( + POST_TRANSITION, + PRE_TRANSITION, + create_pendulum_time, + to_timezone, +) from dagster._utils.schedules import ( _croniter_string_iterator, cron_string_iterator, @@ -77,7 +82,7 @@ def test_cron_iterator_leap_day(): create_pendulum_time(2023, 10, 27, 2, 0, 0, tz="Europe/Berlin"), # +2:00 create_pendulum_time(2023, 10, 28, 2, 0, 0, tz="Europe/Berlin"), # +2:00 create_pendulum_time( - 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), # +1:00 create_pendulum_time(2023, 10, 30, 2, 0, 0, tz="Europe/Berlin"), # +1:00 create_pendulum_time(2023, 10, 31, 2, 0, 0, tz="Europe/Berlin"), # +1:00 @@ -89,22 +94,22 @@ def test_cron_iterator_leap_day(): "30 2 * * *", [ create_pendulum_time( - 2023, 10, 27, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 27, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 28, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 28, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 30, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 30, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 31, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 31, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 11, 1, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 11, 1, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), ], ), @@ -128,10 +133,10 @@ def test_cron_iterator_leap_day(): create_pendulum_time(2023, 10, 29, 0, 45, 0, tz="Europe/Berlin"), create_pendulum_time(2023, 10, 29, 1, 45, 0, tz="Europe/Berlin"), create_pendulum_time( - 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=pendulum.PRE_TRANSITION + 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=PRE_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time(2023, 10, 29, 3, 45, 0, tz="Europe/Berlin"), ], @@ -143,10 +148,10 @@ def test_cron_iterator_leap_day(): create_pendulum_time(2023, 10, 29, 0, 0, 0, tz="Europe/Berlin"), create_pendulum_time(2023, 10, 29, 1, 0, 0, tz="Europe/Berlin"), create_pendulum_time( - 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.PRE_TRANSITION + 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=PRE_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time(2023, 10, 29, 3, 0, 0, tz="Europe/Berlin"), ], @@ -169,22 +174,22 @@ def test_cron_iterator_leap_day(): "0 2 * * 0", # Every sunday at 2 AM [ create_pendulum_time( - 2023, 10, 15, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 15, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 22, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 22, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 11, 5, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 11, 5, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 11, 12, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 11, 12, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 11, 19, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 11, 19, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), ], ), @@ -193,22 +198,22 @@ def test_cron_iterator_leap_day(): "30 2 * * 0", # Every sunday at 2:30 AM [ create_pendulum_time( - 2023, 10, 15, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 15, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 22, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 22, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 11, 5, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 11, 5, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 11, 12, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 11, 12, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 11, 19, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 11, 19, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), ], ), @@ -240,63 +245,33 @@ def test_cron_iterator_leap_day(): "US/Central", "0 1 5 * *", # 5th of each month at 1AM [ - create_pendulum_time( - 2023, 9, 5, 1, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 10, 5, 1, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 11, 5, 1, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 12, 5, 1, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2024, 1, 5, 1, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), + create_pendulum_time(2023, 9, 5, 1, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 10, 5, 1, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 11, 5, 1, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 12, 5, 1, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2024, 1, 5, 1, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), ], ), ( "US/Central", "30 1 5 * *", # 5th of each month at 130AM [ - create_pendulum_time( - 2023, 9, 5, 1, 30, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 10, 5, 1, 30, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 11, 5, 1, 30, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 12, 5, 1, 30, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2024, 1, 5, 1, 30, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), + create_pendulum_time(2023, 9, 5, 1, 30, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 10, 5, 1, 30, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 11, 5, 1, 30, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 12, 5, 1, 30, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2024, 1, 5, 1, 30, 0, tz="US/Central", dst_rule=POST_TRANSITION), ], ), ( "US/Central", "0 2 5 * *", # 5th of each month at 2AM [ - create_pendulum_time( - 2023, 9, 5, 2, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 10, 5, 2, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 11, 5, 2, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2023, 12, 5, 2, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), - create_pendulum_time( - 2024, 1, 5, 2, 0, 0, tz="US/Central", dst_rule=pendulum.POST_TRANSITION - ), + create_pendulum_time(2023, 9, 5, 2, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 10, 5, 2, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 11, 5, 2, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2023, 12, 5, 2, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), + create_pendulum_time(2024, 1, 5, 2, 0, 0, tz="US/Central", dst_rule=POST_TRANSITION), ], ), # Daily / spring forward @@ -507,28 +482,28 @@ def test_cron_iterator_leap_day(): create_pendulum_time(2023, 10, 29, 1, 30, 0, tz="Europe/Berlin"), create_pendulum_time(2023, 10, 29, 1, 45, 0, tz="Europe/Berlin"), create_pendulum_time( - 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.PRE_TRANSITION + 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=PRE_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 15, 0, tz="Europe/Berlin", dst_rule=pendulum.PRE_TRANSITION + 2023, 10, 29, 2, 15, 0, tz="Europe/Berlin", dst_rule=PRE_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.PRE_TRANSITION + 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=PRE_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=pendulum.PRE_TRANSITION + 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=PRE_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 0, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 15, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 15, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 30, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time( - 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=pendulum.POST_TRANSITION + 2023, 10, 29, 2, 45, 0, tz="Europe/Berlin", dst_rule=POST_TRANSITION ), create_pendulum_time(2023, 10, 29, 3, 0, 0, tz="Europe/Berlin"), create_pendulum_time(2023, 10, 29, 3, 15, 0, tz="Europe/Berlin"), diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py b/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py index 63541a55d9eeb..77e93be77eee4 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/test_pythonic_resources.py @@ -29,7 +29,7 @@ from dagster._core.types.loadable_target_origin import LoadableTargetOrigin from dagster._core.workspace.context import WorkspaceProcessContext from dagster._core.workspace.load_target import ModuleTarget -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from .test_scheduler_run import evaluate_schedules, validate_tick, wait_for_all_runs_to_start @@ -190,7 +190,7 @@ def test_resources( "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo_struct_resources.get_external_schedule(schedule_name) instance.start_schedule(external_schedule) @@ -201,7 +201,7 @@ def test_resources( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(seconds=30) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context_struct_resources, None, pendulum.now("UTC")) wait_for_all_runs_to_start(instance) diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_failure_recovery.py b/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_failure_recovery.py index 585c671d48169..b944e2dcc8f49 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_failure_recovery.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_failure_recovery.py @@ -18,7 +18,7 @@ get_crash_signals, ) from dagster._seven import IS_WINDOWS -from dagster._seven.compat.pendulum import create_pendulum_time +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time from dagster._utils import DebugCrashFlags, get_terminate_signal from .conftest import workspace_load_target @@ -61,7 +61,7 @@ def _test_launch_scheduled_runs_in_subprocess( with create_test_daemon_workspace_context( workspace_load_target(), instance ) as workspace_context: - with pendulum.test(execution_datetime): + with pendulum_freeze_time(execution_datetime): evaluate_schedules( workspace_context, executor, @@ -91,7 +91,7 @@ def test_failure_recovery_before_run_created( freeze_datetime = initial_datetime.add() external_schedule = external_repo.get_external_schedule("simple_schedule") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_schedule(external_schedule) debug_crash_flags = {external_schedule.name: {crash_location: crash_signal}} @@ -114,7 +114,7 @@ def test_failure_recovery_before_run_created( assert instance.get_runs_count() == 0 freeze_datetime = freeze_datetime.add(minutes=5) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_process = spawn_ctx.Process( target=_test_launch_scheduled_runs_in_subprocess, args=[instance.get_ref(), freeze_datetime, None, executor], @@ -161,7 +161,7 @@ def test_failure_recovery_after_run_created( initial_datetime = create_pendulum_time(year=2019, month=2, day=27, hour=0, minute=0, second=0) freeze_datetime = initial_datetime.add() external_schedule = external_repo.get_external_schedule("simple_schedule") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_schedule(external_schedule) debug_crash_flags = {external_schedule.name: {crash_location: crash_signal}} @@ -205,7 +205,7 @@ def test_failure_recovery_after_run_created( validate_run_exists(instance.get_runs()[0], freeze_datetime) freeze_datetime = freeze_datetime.add(minutes=5) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Running again just launches the existing run and marks the tick as success scheduler_process = spawn_ctx.Process( target=_test_launch_scheduled_runs_in_subprocess, @@ -248,7 +248,7 @@ def test_failure_recovery_after_tick_success( initial_datetime = create_pendulum_time(year=2019, month=2, day=27, hour=0, minute=0, second=0) freeze_datetime = initial_datetime.add() external_schedule = external_repo.get_external_schedule("simple_schedule") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_schedule(external_schedule) debug_crash_flags = {external_schedule.name: {crash_location: crash_signal}} @@ -289,7 +289,7 @@ def test_failure_recovery_after_tick_success( ) freeze_datetime = freeze_datetime.add(minutes=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Running again just marks the tick as success since the run has already started scheduler_process = spawn_ctx.Process( target=_test_launch_scheduled_runs_in_subprocess, @@ -331,7 +331,7 @@ def test_failure_recovery_between_multi_runs( initial_datetime = create_pendulum_time(year=2019, month=2, day=28, hour=0, minute=0, second=0) freeze_datetime = initial_datetime.add() external_schedule = external_repo.get_external_schedule("multi_run_schedule") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_schedule(external_schedule) debug_crash_flags = {external_schedule.name: {crash_location: crash_signal}} @@ -355,7 +355,7 @@ def test_failure_recovery_between_multi_runs( assert len(ticks) == 1 freeze_datetime = freeze_datetime.add(minutes=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_process = spawn_ctx.Process( target=_test_launch_scheduled_runs_in_subprocess, args=[instance.get_ref(), freeze_datetime, None, executor], diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_run.py b/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_run.py index 7869b306de787..4126d5425d71e 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_run.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_run.py @@ -64,7 +64,7 @@ launch_scheduled_runs_for_schedule_iterator, ) from dagster._seven import wait_for_process -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from dagster._utils import DebugCrashFlags, find_free_port from dagster._utils.error import SerializableErrorInfo from dagster._utils.partitions import DEFAULT_DATE_FORMAT @@ -689,7 +689,7 @@ def test_error_load_code_location(instance: DagsterInstance, executor: ThreadPoo ) as workspace_context: fake_origin = _get_unloadable_schedule_origin() freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): schedule_state = InstigatorState( fake_origin, InstigatorType.SCHEDULE, @@ -699,7 +699,7 @@ def test_error_load_code_location(instance: DagsterInstance, executor: ThreadPoo instance.add_instigator_state(schedule_state) freeze_datetime = freeze_datetime.add(seconds=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 0 @@ -709,7 +709,7 @@ def test_error_load_code_location(instance: DagsterInstance, executor: ThreadPoo assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 0 ticks = instance.get_ticks(fake_origin.get_id(), schedule_state.selector_id) @@ -741,7 +741,7 @@ def test_grpc_server_down(instance: DagsterInstance, executor: ThreadPoolExecuto instance, ) ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("simple_schedule") instance.start_schedule(external_schedule) # freeze the working workspace snapshot @@ -850,7 +850,7 @@ def test_status_in_code_schedule(instance: DagsterInstance, executor: ThreadPool assert code_location external_repo = code_location.get_repository("the_status_in_code_repo") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): running_schedule = external_repo.get_external_schedule("always_running_schedule") not_running_schedule = external_repo.get_external_schedule("never_running_schedule") @@ -938,7 +938,7 @@ def test_status_in_code_schedule(instance: DagsterInstance, executor: ThreadPool assert current_state.instigator_data == reset_instigator_state.instigator_data freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 1 @@ -981,7 +981,7 @@ def test_status_in_code_schedule(instance: DagsterInstance, executor: ThreadPool assert ticks[0].status == TickStatus.SUCCESS freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 2 ticks = instance.get_ticks(always_running_origin.get_id(), running_schedule.selector_id) @@ -990,7 +990,7 @@ def test_status_in_code_schedule(instance: DagsterInstance, executor: ThreadPool # Now try with an error workspace - the job state should not be deleted # since its associated with an errored out location - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): workspace_context._location_entry_dict[ # noqa: SLF001 "test_location" ] = workspace_context._location_entry_dict["test_location"]._replace( # noqa: SLF001 @@ -1008,7 +1008,7 @@ def test_status_in_code_schedule(instance: DagsterInstance, executor: ThreadPool with create_test_daemon_workspace_context( EmptyWorkspaceTarget(), instance ) as empty_workspace_ctx: - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(empty_workspace_ctx, executor, pendulum.now("UTC")) ticks = instance.get_ticks(always_running_origin.get_id(), running_schedule.selector_id) assert len(ticks) == 2 @@ -1046,7 +1046,7 @@ def test_change_default_status(instance: DagsterInstance, executor: ThreadPoolEx instance.add_instigator_state(schedule_state) freeze_datetime = freeze_datetime.add(days=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Traveling two more days in the future before running results in two new ticks evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -1094,7 +1094,7 @@ def test_repository_namespacing(instance: DagsterInstance, executor): workspace_load_target=workspace_load_target(attribute=None), # load all repos instance=instance, ) as full_workspace_context: - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): full_location = cast( CodeLocation, next( @@ -1144,7 +1144,7 @@ def test_repository_namespacing(instance: DagsterInstance, executor): assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(full_workspace_context, executor, pendulum.now("UTC")) assert ( @@ -1187,7 +1187,7 @@ def test_stale_request_context( external_repo: ExternalRepository, ): freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("many_requests_schedule") schedule_origin = external_schedule.get_external_origin() @@ -1249,7 +1249,7 @@ def test_launch_failure( schedule_origin = external_schedule.get_external_origin() freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): exploding_ctx = workspace_context.copy_for_test_instance(scheduler_instance) scheduler_instance.start_schedule(external_schedule) @@ -1306,7 +1306,7 @@ def test_schedule_mutation( assert schedule_one.selector_id == schedule_two.selector_id freeze_datetime = create_pendulum_time(year=2023, month=2, day=1, tz="UTC") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # start the schedule at 12:00 AM, it is scheduled to go at 2:00 AM instance.start_schedule(schedule_one) evaluate_schedules(workspace_one, executor, pendulum.now("UTC")) @@ -1315,7 +1315,7 @@ def test_schedule_mutation( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(hours=1, minutes=59) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # now check the schedule at 1:59 AM, where the schedule is not to fire until 2:00 AM evaluate_schedules(workspace_one, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 0 @@ -1323,7 +1323,7 @@ def test_schedule_mutation( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(minutes=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Now change the schedule to be at 1:00 AM. It should not generate a tick because we last # evaluated at 1:59AM and it is now 2:00 AM. We expect the new schedule to wait until # tomorrow to create a new tick. @@ -1333,7 +1333,7 @@ def test_schedule_mutation( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(hours=23) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_two, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 1 ticks = instance.get_ticks(origin_two.get_id(), schedule_two.selector_id) @@ -1354,7 +1354,7 @@ def test_simple_schedule( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("simple_schedule") schedule_origin = external_schedule.get_external_origin() @@ -1377,7 +1377,7 @@ def test_simple_schedule( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 @@ -1415,7 +1415,7 @@ def test_simple_schedule( # Verify advancing in time but not going past a tick doesn't add any new runs freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 @@ -1426,7 +1426,7 @@ def test_simple_schedule( assert ticks[0].status == TickStatus.SUCCESS freeze_datetime = freeze_datetime.add(days=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Traveling two more days in the future passes two ticks times, but only the most recent # will be created as a tick with a corresponding run. evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -1473,7 +1473,7 @@ def test_schedule_with_different_origin( ) freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): schedule_state = InstigatorState( modified_origin, InstigatorType.SCHEDULE, @@ -1486,7 +1486,7 @@ def test_schedule_with_different_origin( freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 @@ -1504,7 +1504,7 @@ def test_old_tick_schedule( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("simple_schedule") # Create an old tick from several days ago @@ -1523,7 +1523,7 @@ def test_old_tick_schedule( scheduler_instance.start_schedule(external_schedule) freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 @@ -1569,7 +1569,7 @@ def test_schedule_without_timezone( year=2019, month=2, day=27, hour=0, minute=0, second=0, tz="UTC" ) - with pendulum.test(initial_datetime): + with pendulum_freeze_time(initial_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -1620,7 +1620,7 @@ def test_bad_eval_fn_no_retries( freeze_datetime = create_pendulum_time( year=2019, month=2, day=27, hour=0, minute=0, second=0 ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -1661,7 +1661,7 @@ def test_bad_eval_fn_no_retries( ) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 0 @@ -1693,7 +1693,7 @@ def test_invalid_eval_fn_with_retries( freeze_datetime = create_pendulum_time( year=2019, month=2, day=27, hour=0, minute=0, second=0 ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC"), max_tick_retries=2) @@ -1751,7 +1751,7 @@ def test_invalid_eval_fn_with_retries( ) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 0 @@ -1788,7 +1788,7 @@ def test_passes_on_retry( freeze_datetime = create_pendulum_time( year=2019, month=2, day=27, hour=0, minute=0, second=0 ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC"), max_tick_retries=1) @@ -1827,7 +1827,7 @@ def test_passes_on_retry( ) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC"), max_tick_retries=1) assert scheduler_instance.get_runs_count() == 2 @@ -1863,7 +1863,7 @@ def test_bad_should_execute( minute=0, second=0, ) - with pendulum.test(initial_datetime): + with pendulum_freeze_time(initial_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -1896,7 +1896,7 @@ def test_skip( external_schedule = external_repo.get_external_schedule("skip_schedule") schedule_origin = external_schedule.get_external_origin() freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -1928,7 +1928,7 @@ def test_wrong_config_schedule( freeze_datetime = create_pendulum_time( year=2019, month=2, day=27, hour=0, minute=0, second=0 ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -1962,7 +1962,7 @@ def test_schedule_run_default_config( initial_datetime = create_pendulum_time( year=2019, month=2, day=27, hour=0, minute=0, second=0 ) - with pendulum.test(initial_datetime): + with pendulum_freeze_time(initial_datetime): scheduler_instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -2011,7 +2011,7 @@ def test_bad_schedules_mixed_with_good_schedule( bad_origin = bad_schedule.get_external_origin() unloadable_origin = _get_unloadable_schedule_origin() freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(good_schedule) scheduler_instance.start_schedule(bad_schedule) @@ -2061,7 +2061,7 @@ def test_bad_schedules_mixed_with_good_schedule( assert len(unloadable_ticks) == 0 freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): new_now = pendulum.now("UTC") evaluate_schedules(workspace_context, executor, new_now) @@ -2127,7 +2127,7 @@ def test_run_scheduled_on_time_boundary( schedule_origin = external_schedule.get_external_origin() freeze_datetime = feb_27_2019_start_of_day() # 00:00:00 - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Start schedule exactly at midnight scheduler_instance.start_schedule(external_schedule) @@ -2150,7 +2150,7 @@ def test_bad_load_repository( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("simple_schedule") valid_schedule_origin = external_schedule.get_external_origin() @@ -2172,7 +2172,7 @@ def test_bad_load_repository( scheduler_instance.add_instigator_state(schedule_state) initial_datetime = freeze_datetime.add(seconds=1) - with pendulum.test(initial_datetime): + with pendulum_freeze_time(initial_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 0 @@ -2198,7 +2198,7 @@ def test_bad_load_schedule( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("simple_schedule") valid_schedule_origin = external_schedule.get_external_origin() @@ -2217,7 +2217,7 @@ def test_bad_load_schedule( scheduler_instance.add_instigator_state(schedule_state) initial_datetime = freeze_datetime.add(seconds=1) - with pendulum.test(initial_datetime): + with pendulum_freeze_time(initial_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 0 @@ -2246,7 +2246,7 @@ def test_load_code_location_not_in_workspace( "US/Central", ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("simple_schedule") valid_schedule_origin = external_schedule.get_external_origin() @@ -2273,7 +2273,7 @@ def test_load_code_location_not_in_workspace( scheduler_instance.add_instigator_state(schedule_state) initial_datetime = freeze_datetime.add(seconds=1) - with pendulum.test(initial_datetime): + with pendulum_freeze_time(initial_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 0 @@ -2300,12 +2300,12 @@ def test_multiple_schedules_on_different_time_ranges( external_schedule = external_repo.get_external_schedule("simple_schedule") external_hourly_schedule = external_repo.get_external_schedule("simple_hourly_schedule") freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) scheduler_instance.start_schedule(external_hourly_schedule) freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 2 @@ -2323,7 +2323,7 @@ def test_multiple_schedules_on_different_time_ranges( assert hourly_ticks[0].status == TickStatus.SUCCESS freeze_datetime = freeze_datetime.add(hours=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 3 @@ -2351,13 +2351,13 @@ def test_union_schedule( ): # This is a Wednesday. freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("union_schedule") schedule_origin = external_schedule.get_external_origin() scheduler_instance.start_schedule(external_schedule) # No new runs should be launched - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 0 @@ -2367,7 +2367,7 @@ def test_union_schedule( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 @@ -2394,7 +2394,7 @@ def test_union_schedule( ) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 2 @@ -2420,7 +2420,7 @@ def test_union_schedule( ) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 3 @@ -2448,7 +2448,7 @@ def test_union_schedule( # No new runs should be launched freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 3 @@ -2466,7 +2466,7 @@ def test_multi_runs( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("multi_run_schedule") schedule_origin = external_schedule.get_external_origin() scheduler_instance.start_schedule(external_schedule) @@ -2486,7 +2486,7 @@ def test_multi_runs( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 2 ticks = scheduler_instance.get_ticks( @@ -2524,7 +2524,7 @@ def test_multi_runs( assert ticks[0].status == TickStatus.SUCCESS freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Traveling one more day in the future before running results in a tick evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 4 @@ -2544,7 +2544,7 @@ def test_multi_run_list( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("multi_run_list_schedule") schedule_origin = external_schedule.get_external_origin() scheduler_instance.start_schedule(external_schedule) @@ -2564,7 +2564,7 @@ def test_multi_run_list( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 2 ticks = scheduler_instance.get_ticks( @@ -2602,7 +2602,7 @@ def test_multi_run_list( assert ticks[0].status == TickStatus.SUCCESS freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Traveling one more day in the future before running results in a tick evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 4 @@ -2622,7 +2622,7 @@ def test_multi_runs_missing_run_key( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule( "multi_run_schedule_with_missing_run_key" ) @@ -2656,14 +2656,14 @@ def test_large_schedule( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_one_second_to_midnight() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("large_schedule") schedule_origin = external_schedule.get_external_origin() scheduler_instance.start_schedule(external_schedule) freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 @@ -2681,7 +2681,7 @@ def test_skip_reason_schedule( executor: ThreadPoolExecutor, ): freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("empty_schedule") schedule_origin = external_schedule.get_external_origin() @@ -2715,7 +2715,7 @@ def test_many_requests_schedule( submit_executor: Optional[ThreadPoolExecutor], ): freeze_datetime = feb_27_2019_start_of_day() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("many_requests_schedule") schedule_origin = external_schedule.get_external_origin() @@ -2753,7 +2753,7 @@ def test_asset_selection( external_schedule = external_repo.get_external_schedule("asset_selection_schedule") schedule_origin = external_schedule.get_external_origin() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) ticks = scheduler_instance.get_ticks( @@ -2765,7 +2765,7 @@ def test_asset_selection( scheduler_instance.get_ticks(schedule_origin.get_id(), external_schedule.selector_id) freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 @@ -2803,12 +2803,12 @@ def test_stale_asset_selection_never_materialized( freeze_datetime = feb_27_2019_one_second_to_midnight() external_schedule = external_repo.get_external_schedule("stale_asset_selection_schedule") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) # never materialized so all assets stale freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(scheduler_instance) schedule_run = next( @@ -2831,14 +2831,14 @@ def test_stale_asset_selection_empty( freeze_datetime = feb_27_2019_one_second_to_midnight() external_schedule = external_repo.get_external_schedule("stale_asset_selection_schedule") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) materialize([asset1, asset2], instance=scheduler_instance) # assets previously materialized so we expect empy set freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(scheduler_instance) schedule_run = next( @@ -2857,14 +2857,14 @@ def test_stale_asset_selection_subset( freeze_datetime = feb_27_2019_one_second_to_midnight() external_schedule = external_repo.get_external_schedule("stale_asset_selection_schedule") - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) materialize([asset1], instance=scheduler_instance) # assets previously materialized so we expect empy set freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(scheduler_instance) schedule_run = next( @@ -2884,7 +2884,7 @@ def test_source_asset_observation( external_schedule = external_repo.get_external_schedule("source_asset_observation_schedule") schedule_origin = external_schedule.get_external_origin() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): scheduler_instance.start_schedule(external_schedule) ticks = scheduler_instance.get_ticks( @@ -2896,7 +2896,7 @@ def test_source_asset_observation( scheduler_instance.get_ticks(schedule_origin.get_id(), external_schedule.selector_id) freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert scheduler_instance.get_runs_count() == 1 diff --git a/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_timezones.py b/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_timezones.py index 63f3048af066d..4651af998fdc1 100644 --- a/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_timezones.py +++ b/python_modules/dagster/dagster_tests/scheduler_tests/test_scheduler_timezones.py @@ -6,7 +6,7 @@ from dagster._core.instance import DagsterInstance from dagster._core.scheduler.instigation import TickStatus from dagster._core.workspace.context import WorkspaceProcessContext -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone from .test_scheduler_run import ( evaluate_schedules, @@ -28,7 +28,7 @@ def test_non_utc_timezone_run( freeze_datetime = to_timezone( create_pendulum_time(2019, 2, 27, 23, 59, 59, tz="US/Central"), "US/Pacific" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("daily_central_time_schedule") schedule_origin = external_schedule.get_external_origin() @@ -46,7 +46,7 @@ def test_non_utc_timezone_run( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 1 @@ -91,7 +91,7 @@ def test_differing_timezones( freeze_datetime = to_timezone( create_pendulum_time(2019, 2, 27, 23, 59, 59, tz="US/Eastern"), "US/Pacific" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("daily_central_time_schedule") external_eastern_schedule = external_repo.get_external_schedule( "daily_eastern_time_schedule" @@ -120,7 +120,7 @@ def test_differing_timezones( # Past midnight eastern time, the eastern timezone schedule will run, but not the central timezone freeze_datetime = freeze_datetime.add(minutes=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 1 @@ -151,7 +151,7 @@ def test_differing_timezones( # Past midnight central time, the central timezone schedule will now run freeze_datetime = freeze_datetime.add(hours=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 2 @@ -204,7 +204,7 @@ def test_different_days_in_different_timezones( freeze_datetime = to_timezone( create_pendulum_time(2019, 2, 27, 22, 59, 59, tz="US/Central"), "US/Pacific" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): # Runs every day at 11PM (CST) external_schedule = external_repo.get_external_schedule("daily_late_schedule") schedule_origin = external_schedule.get_external_origin() @@ -220,7 +220,7 @@ def test_different_days_in_different_timezones( assert len(ticks) == 0 freeze_datetime = freeze_datetime.add(seconds=2) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) assert instance.get_runs_count() == 1 @@ -269,7 +269,7 @@ def test_hourly_dst_spring_forward( external_schedule = external_repo.get_external_schedule("hourly_central_time_schedule") schedule_origin = external_schedule.get_external_origin() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -278,13 +278,13 @@ def test_hourly_dst_spring_forward( assert len(ticks) == 1 freeze_datetime = freeze_datetime.add(hours=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) # DST has now happened, 2 hours later it is 4AM CST # Should be 3 runs: 1AM CST, 3AM CST, 4AM CST freeze_datetime = freeze_datetime.add(hours=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(instance) @@ -336,7 +336,7 @@ def test_hourly_dst_fall_back( external_schedule = external_repo.get_external_schedule("hourly_central_time_schedule") schedule_origin = external_schedule.get_external_origin() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -346,13 +346,13 @@ def test_hourly_dst_fall_back( for _ in range(3): freeze_datetime = freeze_datetime.add(hours=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) # DST has now happened, 4 hours later it is 3:30AM CST # Should be 4 runs: 1AM CDT, 2AM CDT, 2AM CST, 3AM CST freeze_datetime = freeze_datetime.add(hours=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(instance) @@ -417,7 +417,7 @@ def test_daily_dst_spring_forward( external_schedule = external_repo.get_external_schedule("daily_central_time_schedule") schedule_origin = external_schedule.get_external_origin() - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): instance.start_schedule(external_schedule) evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) @@ -426,11 +426,11 @@ def test_daily_dst_spring_forward( assert len(ticks) == 1 freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(instance) @@ -482,7 +482,7 @@ def test_daily_dst_fall_back( create_pendulum_time(2019, 11, 3, 0, 0, 0, tz="US/Central"), "US/Pacific" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule("daily_central_time_schedule") schedule_origin = external_schedule.get_external_origin() instance.start_schedule(external_schedule) @@ -493,11 +493,11 @@ def test_daily_dst_fall_back( assert len(ticks) == 1 freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(instance) @@ -550,7 +550,7 @@ def test_execute_during_dst_transition_spring_forward( create_pendulum_time(2019, 3, 9, 0, 0, 0, tz="US/Central"), "US/Pacific" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule( "daily_dst_transition_schedule_skipped_time" ) @@ -564,11 +564,11 @@ def test_execute_during_dst_transition_spring_forward( for _ in range(4): freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(instance) @@ -620,7 +620,7 @@ def test_execute_during_dst_transition_fall_back( create_pendulum_time(2019, 11, 2, 0, 0, 0, tz="US/Central"), "US/Pacific" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): external_schedule = external_repo.get_external_schedule( "daily_dst_transition_schedule_doubled_time" ) @@ -634,11 +634,11 @@ def test_execute_during_dst_transition_fall_back( for _ in range(2): freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) freeze_datetime = freeze_datetime.add(days=1) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): evaluate_schedules(workspace_context, executor, pendulum.now("UTC")) wait_for_all_runs_to_start(instance) diff --git a/python_modules/dagster/dagster_tests/storage_tests/utils/run_storage.py b/python_modules/dagster/dagster_tests/storage_tests/utils/run_storage.py index 7d587d87714d7..e126422525f96 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/utils/run_storage.py +++ b/python_modules/dagster/dagster_tests/storage_tests/utils/run_storage.py @@ -46,7 +46,7 @@ from dagster._daemon.daemon import SensorDaemon from dagster._daemon.types import DaemonHeartbeat from dagster._serdes import serialize_pp -from dagster._seven.compat.pendulum import create_pendulum_time, to_timezone +from dagster._seven.compat.pendulum import create_pendulum_time, pendulum_freeze_time, to_timezone win_py36 = _seven.IS_WINDOWS and sys.version_info[0] == 3 and sys.version_info[1] == 6 @@ -1395,7 +1395,7 @@ def my_job(): create_pendulum_time(2019, 11, 2, 0, 0, 0, tz="US/Central"), "US/Pacific" ) - with pendulum.test(freeze_datetime): + with pendulum_freeze_time(freeze_datetime): result = my_job.execute_in_process(instance=instance) records = instance.get_run_records(filters=RunsFilter(run_ids=[result.run_id])) assert len(records) == 1 diff --git a/python_modules/dagster/setup.py b/python_modules/dagster/setup.py index df0ce1d57b551..b7519a27cc015 100644 --- a/python_modules/dagster/setup.py +++ b/python_modules/dagster/setup.py @@ -86,7 +86,7 @@ def get_version() -> str: f"grpcio>={GRPC_VERSION_FLOOR}", f"grpcio-health-checking>={GRPC_VERSION_FLOOR}", "packaging>=20.9", - "pendulum>=0.7.0,<3", + "pendulum>=0.7.0,<4", "protobuf>=3.20.0,<5", # min protobuf version to be compatible with both protobuf 3 and 4 "python-dateutil", "python-dotenv", diff --git a/python_modules/dagster/tox.ini b/python_modules/dagster/tox.ini index c791f8d736352..214e275d50c90 100644 --- a/python_modules/dagster/tox.ini +++ b/python_modules/dagster/tox.ini @@ -9,8 +9,12 @@ setenv = STRICT_GRPC_SERVER_PROCESS_WAIT = "1" passenv = CI_* COVERALLS_REPO_TOKEN AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID BUILDKITE* DAGSTER_DOCKER_* GRPC_SERVER_HOST deps = - scheduler_tests_old_pendulum: pendulum<2 - definitions_tests_old_pendulum: pendulum<2 + scheduler_tests_pendulum_1: pendulum<2 + scheduler_tests_pendulum_2: pendulum>1,<3 + scheduler_tests: pendulum>2 + definitions_tests_pendulum_1: pendulum<2 + definitions_tests_pendulum_2: pendulum>1,<3 + definitions_tests: pendulum>2 storage_tests_sqlalchemy_1_3: sqlalchemy<1.4 storage_tests_sqlalchemy_1_4: sqlalchemy<2 general_tests_old_protobuf: protobuf<4 @@ -32,14 +36,16 @@ commands = type_signature_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/core_tests {env:COVERAGE_ARGS} --durations 10 {posargs} -m 'typesignature' storage_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/storage_tests {env:COVERAGE_ARGS} --durations 10 {posargs} definitions_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/definitions_tests {env:COVERAGE_ARGS} --durations 10 {posargs} - definitions_tests_old_pendulum: pytest -c ../../pyproject.toml -vv ./dagster_tests/definitions_tests {env:COVERAGE_ARGS} --durations 10 {posargs} + definitions_tests_pendulum_1: pytest -c ../../pyproject.toml -vv ./dagster_tests/definitions_tests {env:COVERAGE_ARGS} --durations 10 {posargs} + definitions_tests_pendulum_2: pytest -c ../../pyproject.toml -vv ./dagster_tests/definitions_tests {env:COVERAGE_ARGS} --durations 10 {posargs} asset_defs_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/asset_defs_tests {env:COVERAGE_ARGS} --durations 10 {posargs} storage_tests_sqlalchemy_1_3: pytest -c ../../pyproject.toml -vv ./dagster_tests/storage_tests {env:COVERAGE_ARGS} --durations 10 {posargs} storage_tests_sqlalchemy_1_4: pytest -c ../../pyproject.toml -vv ./dagster_tests/storage_tests {env:COVERAGE_ARGS} --durations 10 {posargs} daemon_sensor_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/daemon_sensor_tests {env:COVERAGE_ARGS} --durations 10 {posargs} daemon_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/daemon_tests {env:COVERAGE_ARGS} --durations 10 {posargs} scheduler_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/scheduler_tests {env:COVERAGE_ARGS} --durations 10 {posargs} - scheduler_tests_old_pendulum: pytest -c ../../pyproject.toml -vv ./dagster_tests/scheduler_tests {env:COVERAGE_ARGS} --durations 10 {posargs} + scheduler_tests_pendulum_1: pytest -c ../../pyproject.toml -vv ./dagster_tests/scheduler_tests {env:COVERAGE_ARGS} --durations 10 {posargs} + scheduler_tests_pendulum_2: pytest -c ../../pyproject.toml -vv ./dagster_tests/scheduler_tests {env:COVERAGE_ARGS} --durations 10 {posargs} general_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/general_tests {env:COVERAGE_ARGS} --durations 10 {posargs} general_tests_old_protobuf: pytest -c ../../pyproject.toml -vv ./dagster_tests/general_tests {env:COVERAGE_ARGS} --durations 10 {posargs} execution_tests: pytest -c ../../pyproject.toml -vv ./dagster_tests/execution_tests {env:COVERAGE_ARGS} --durations 10 {posargs}