From 2841c9075c050570b9fd95bc13c533f04b4e4a82 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 28 Sep 2022 17:34:34 +0100 Subject: [PATCH] Bugfix: avoid overflow --- .../databases/main/event_federation.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index b6bcff7262b2..9de77b3e86a1 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -73,13 +73,20 @@ logger = logging.getLogger(__name__) -BACKFILL_EVENT_BACKOFF_UPPER_BOUND_SECONDS: int = int( - datetime.timedelta(days=7).total_seconds() -) +BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS = 8 BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_SECONDS: int = int( datetime.timedelta(hours=1).total_seconds() ) - +# The longest backoff period is then: +# _LONGEST_BACKOFF_PERIOD_SECONDS = ( +# (1 << BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS) +# * BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_SECONDS, +# ) +# which is 2 ** 8 hours = 256 hours ≈ 10 days. This number needs to fit +# in a 32 bit signed int, or else Postgres will error. +# assert _LONGEST_BACKOFF_PERIOD_SECONDS < ((2 ** 31) - 1) +# (We could use a bigint, but bigint overflows still cause errors. This +# at least avoids CASTing in the queries below.) # All the info we need while iterating the DAG while backfilling @attr.s(frozen=True, slots=True, auto_attribs=True) @@ -803,7 +810,10 @@ def get_backfill_points_in_room_txn( */ AND ( failed_backfill_attempt_info.event_id IS NULL - OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + {least_function}((1 << failed_backfill_attempt_info.num_attempts) * ? /* step */, ? /* upper bound */) + OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + ( + (1 << {least_function}(failed_backfill_attempt_info.num_attempts, ? /* max doubling steps */)) + * ? /* step */ + ) ) /** * Sort from highest to the lowest depth. Then tie-break on @@ -819,8 +829,8 @@ def get_backfill_points_in_room_txn( room_id, False, self._clock.time_msec(), + BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS, 1000 * BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_SECONDS, - 1000 * BACKFILL_EVENT_BACKOFF_UPPER_BOUND_SECONDS, ), ) @@ -888,7 +898,10 @@ def get_insertion_event_backward_extremities_in_room_txn( */ AND ( failed_backfill_attempt_info.event_id IS NULL - OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + {least_function}((1 << failed_backfill_attempt_info.num_attempts) * ? /* step */, ? /* upper bound */) + OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + ( + (1 << {least_function}(failed_backfill_attempt_info.num_attempts, ? /* max doubling steps */)) + * ? /* step */ + ) ) /** * Sort from highest to the lowest depth. Then tie-break on @@ -904,8 +917,8 @@ def get_insertion_event_backward_extremities_in_room_txn( ( room_id, self._clock.time_msec(), + BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS, 1000 * BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_SECONDS, - 1000 * BACKFILL_EVENT_BACKOFF_UPPER_BOUND_SECONDS, ), ) return cast(List[Tuple[str, int]], txn.fetchall())