matrix-org · MadLittleMods · Jun 16, 2023 · May 24, 2023 · May 24, 2023 · Jun 8, 2023
@@ -0,0 +1 @@
+Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms.
@@ -92,8 +92,6 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 
 experimental_features:
-  # Enable history backfilling support
-  msc2716_enabled: true
   # client-side support for partial state in /send_join responses
   faster_joins: true
   # Enable support for polls

@@ -244,7 +244,6 @@
             "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
-            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",

@@ -232,7 +232,6 @@ information.
     ^/_matrix/client/v1/rooms/.*/hierarchy$
     ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
     ^/_matrix/client/v1/rooms/.*/threads$
-    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
     ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
     ^/_matrix/client/(r0|v3|unstable)/account/3pid$
     ^/_matrix/client/(r0|v3|unstable)/account/whoami$

@@ -246,10 +246,6 @@ else
   else
     export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
   fi
-
-  # The tests for importing historical messages (MSC2716)
-  # only pass with monoliths, currently.
-  test_tags="$test_tags,msc2716"
 fi
 
 if [[ -n "$ASYNCIO_REACTOR" ]]; then

@@ -123,10 +123,6 @@ class EventTypes:
     SpaceChild: Final = "m.space.child"
     SpaceParent: Final = "m.space.parent"
 
-    MSC2716_INSERTION: Final = "org.matrix.msc2716.insertion"
-    MSC2716_BATCH: Final = "org.matrix.msc2716.batch"
-    MSC2716_MARKER: Final = "org.matrix.msc2716.marker"
-
     Reaction: Final = "m.reaction"
 
 
@@ -222,16 +218,6 @@ class EventContentFields:
     # Used in m.room.guest_access events.
     GUEST_ACCESS: Final = "guest_access"
 
-    # Used on normal messages to indicate they were historically imported after the fact
-    MSC2716_HISTORICAL: Final = "org.matrix.msc2716.historical"
-    # For "insertion" events to indicate what the next batch ID should be in
-    # order to connect to it
-    MSC2716_NEXT_BATCH_ID: Final = "next_batch_id"
-    # Used on "batch" events to indicate which insertion event it connects to
-    MSC2716_BATCH_ID: Final = "batch_id"
-    # For "marker" events
-    MSC2716_INSERTION_EVENT_REFERENCE: Final = "insertion_event_reference"
-
     # The authorising user for joining a restricted room.
     AUTHORISING_USER: Final = "join_authorised_via_users_server"
 

@@ -91,11 +91,6 @@ class RoomVersion:
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
     msc2403_knocking: bool
-    # MSC2716: Adds m.room.power_levels -> content.historical field to control
-    # whether "insertion", "chunk", "marker" events can be sent
-    msc2716_historical: bool
-    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
-    msc2716_redactions: bool
     # MSC3389: Protect relation information from redaction.
     msc3389_relation_redactions: bool
     # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
@@ -130,8 +125,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -153,8 +146,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -176,8 +167,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -199,8 +188,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -222,8 +209,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -245,8 +230,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -268,8 +251,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -291,8 +272,6 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -314,8 +293,6 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -337,8 +314,6 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -360,8 +335,6 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=False,
@@ -383,8 +356,6 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -406,38 +377,13 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
         msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
-    MSC2716v4 = RoomVersion(
-        "org.matrix.msc2716v4",
-        RoomDisposition.UNSTABLE,
-        EventFormatVersions.ROOM_V4_PLUS,
-        StateResolutionVersions.V2,
-        enforce_key_validity=True,
-        special_case_aliases_auth=False,
-        strict_canonicaljson=True,
-        limit_notifications_power_levels=True,
-        msc2175_implicit_room_creator=False,
-        msc2176_redaction_rules=False,
-        msc3083_join_rules=False,
-        msc3375_redaction_rules=False,
-        msc2403_knocking=True,
-        msc2716_historical=True,
-        msc2716_redactions=True,
-        msc3389_relation_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
-        msc3931_push_features=(),
-        msc3989_redaction_rules=False,
-    )
     MSC1767v10 = RoomVersion(
         # MSC1767 (Extensible Events) based on room version "10"
         "org.matrix.msc1767.10",
@@ -453,8 +399,6 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -476,8 +420,6 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -500,8 +442,6 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -526,7 +466,6 @@ class RoomVersions:
         RoomVersions.V9,
         RoomVersions.MSC3787,
         RoomVersions.V10,
-        RoomVersions.MSC2716v4,
         RoomVersions.MSC3989,
         RoomVersions.MSC3820opt2,
     )

@@ -83,7 +83,6 @@
 from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
-from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
@@ -120,7 +119,6 @@ class GenericWorkerStore(
     # the races it creates aren't too bad.
     KeyStore,
     RoomWorkerStore,
-    RoomBatchStore,
     DirectoryWorkerStore,
     PushRulesWorkerStore,
     ApplicationServiceTransactionWorkerStore,

@@ -247,9 +247,6 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None:
         # MSC3026 (busy presence state)
         self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)
 
-        # MSC2716 (importing historical messages)
-        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
-
         # MSC3244 (room version capabilities)
         self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)
 

@@ -339,13 +339,6 @@ def check_state_dependent_auth_rules(
     if event.type == EventTypes.Redaction:
         check_redaction(event.room_version, event, auth_dict)
 
-    if (
-        event.type == EventTypes.MSC2716_INSERTION
-        or event.type == EventTypes.MSC2716_BATCH
-        or event.type == EventTypes.MSC2716_MARKER
-    ):
-        check_historical(event.room_version, event, auth_dict)
-
     logger.debug("Allowing! %s", event)
 
 
@@ -365,7 +358,6 @@ def check_state_dependent_auth_rules(
     RoomVersions.V9,
     RoomVersions.MSC3787,
     RoomVersions.V10,
-    RoomVersions.MSC2716v4,
     RoomVersions.MSC1767v10,
 }
 
@@ -823,38 +815,6 @@ def check_redaction(
     raise AuthError(403, "You don't have permission to redact events")
 
 
-def check_historical(
-    room_version_obj: RoomVersion,
-    event: "EventBase",
-    auth_events: StateMap["EventBase"],
-) -> None:
-    """Check whether the event sender is allowed to send historical related
-    events like "insertion", "batch", and "marker".
-
-    Returns:
-        None
-
-    Raises:
-        AuthError if the event sender is not allowed to send historical related events
-        ("insertion", "batch", and "marker").
-    """
-    # Ignore the auth checks in room versions that do not support historical
-    # events
-    if not room_version_obj.msc2716_historical:
-        return
-
-    user_level = get_user_power_level(event.user_id, auth_events)
-
-    historical_level = get_named_level(auth_events, "historical", 100)
-
-    if user_level < historical_level:
-        raise UnstableSpecAuthError(
-            403,
-            'You don\'t have permission to send send historical related events ("insertion", "batch", and "marker")',
-            errcode=Codes.INSUFFICIENT_POWER,
-        )
-
-
 def _check_power_levels(
     room_version_obj: RoomVersion,
     event: "EventBase",

@@ -198,7 +198,6 @@ def __init__(self, internal_metadata_dict: JsonDict):
     soft_failed: DictProperty[bool] = DictProperty("soft_failed")
     proactively_send: DictProperty[bool] = DictProperty("proactively_send")
     redacted: DictProperty[bool] = DictProperty("redacted")
-    historical: DictProperty[bool] = DictProperty("historical")
 
     txn_id: DictProperty[str] = DictProperty("txn_id")
     """The transaction ID, if it was set when the event was created."""
@@ -288,14 +287,6 @@ def is_redacted(self) -> bool:
         """
         return self._dict.get("redacted", False)
 
-    def is_historical(self) -> bool:
-        """Whether this is a historical message.
-        This is used by the batchsend historical message endpoint and
-        is needed to and mark the event as backfilled and skip some checks
-        like push notifications.
-        """
-        return self._dict.get("historical", False)
-
     def is_notifiable(self) -> bool:
         """Whether this event can trigger a push notification"""
         return not self.is_outlier() or self.is_out_of_band_membership()

@@ -164,21 +164,12 @@ def add_fields(*fields: str) -> None:
         if room_version.msc2176_redaction_rules:
             add_fields("invite")
 
-        if room_version.msc2716_historical:
-            add_fields("historical")
-
     elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth:
         add_fields("aliases")
     elif event_type == EventTypes.RoomHistoryVisibility:
         add_fields("history_visibility")
     elif event_type == EventTypes.Redaction and room_version.msc2176_redaction_rules:
         add_fields("redacts")
-    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_INSERTION:
-        add_fields(EventContentFields.MSC2716_NEXT_BATCH_ID)
-    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_BATCH:
-        add_fields(EventContentFields.MSC2716_BATCH_ID)
-    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_MARKER:
-        add_fields(EventContentFields.MSC2716_INSERTION_EVENT_REFERENCE)
 
     # Protect the rel_type and event_id fields under the m.relates_to field.
     if room_version.msc3389_relation_redactions:
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms.
Copy link Contributor Author MadLittleMods Jun 9, 2023 • edited Loading Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Do we need to worry about gracefully dropping database tables for an experimental feature like this? Relevant tables to drop: -- Drop tables related to MSC2716 since the implementation is being removed DROP TABLE insertion_events; DROP TABLE insertion_event_edges; DROP TABLE insertion_event_extremities; DROP TABLE batch_events; Can we just stop writing to the tables in the Synapse version that this PR ships in, then in the next Synapse version, bump the `SCHEMA_COMPAT_VERSION` and drop the tables? Copy link Member erikjohnston Jun 13, 2023 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. I'd say we may as well? We still want to be careful that we don't accidentally break upgrades/downgrades for people that have turned on experimental features. Easiest way of tracking the task of dropping those tables is to create an issue with the future maintenance label and add it to one of the milestones. Copy link Contributor Author MadLittleMods Jun 15, 2023 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Created #15786 to track this ⏩ MadLittleMods marked this conversation as resolved. Show resolved Hide resolved