Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Redact events in the DB on m.room.redaction #280

Merged
merged 2 commits into from
Sep 6, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/jmoiron/sqlx v1.3.3
github.com/lib/pq v1.10.9
github.com/matrix-org/gomatrixserverlib v0.0.0-20230105074811-965b10ae73ab
github.com/matrix-org/gomatrixserverlib v0.0.0-20230822143230-740f742d6993
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4
github.com/pressly/goose/v3 v3.14.0
github.com/prometheus/client_golang v1.13.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5d
github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
github.com/matrix-org/gomatrixserverlib v0.0.0-20230105074811-965b10ae73ab h1:ChaQdT2mpxMm3GRXNOZzLDQ/wOnlKZ8o60LmZGOjdj8=
github.com/matrix-org/gomatrixserverlib v0.0.0-20230105074811-965b10ae73ab/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4=
github.com/matrix-org/gomatrixserverlib v0.0.0-20230822143230-740f742d6993 h1:88wDfSsqSFyeCnTha8vfK9XJUbNjNXdEZAieOjbBzos=
github.com/matrix-org/gomatrixserverlib v0.0.0-20230822143230-740f742d6993/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk=
github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
Expand Down
34 changes: 33 additions & 1 deletion state/accumulator.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"database/sql"
"encoding/json"
"fmt"

"github.com/matrix-org/sliding-sync/internal"

"github.com/getsentry/sentry-go"
Expand Down Expand Up @@ -398,11 +399,13 @@ func (a *Accumulator) Accumulate(txn *sqlx.Tx, userID, roomID string, prevBatch

var latestNID int64
newEvents := make([]Event, 0, len(eventIDToNID))
var redactTheseEventIDs []string
for _, ev := range dedupedEvents {
nid, ok := eventIDToNID[ev.ID]
if ok {
ev.NID = int64(nid)
if gjson.GetBytes(ev.JSON, "state_key").Exists() {
parsedEv := gjson.ParseBytes(ev.JSON)
if parsedEv.Get("state_key").Exists() {
// XXX: reusing this to mean "it's a state event" as well as "it's part of the state v2 response"
// its important that we don't insert 'ev' at this point as this should be False in the DB.
ev.IsState = true
Expand All @@ -412,11 +415,40 @@ func (a *Accumulator) Accumulate(txn *sqlx.Tx, userID, roomID string, prevBatch
if ev.NID > latestNID {
latestNID = ev.NID
}
// if this is a redaction, try to redact the referenced event (best effort)
if !ev.IsState && ev.Type == "m.room.redaction" {
kegsay marked this conversation as resolved.
Show resolved Hide resolved
// look for top-level redacts then content.redacts (room version 11+)
redactsEventID := parsedEv.Get("redacts").Str
if redactsEventID == "" {
redactsEventID = parsedEv.Get("content.redacts").Str
}
if redactsEventID != "" {
redactTheseEventIDs = append(redactTheseEventIDs, redactsEventID)
}
}
newEvents = append(newEvents, ev)
timelineNIDs = append(timelineNIDs, ev.NID)
}
}

// if we are going to redact things, we need the room version to know the redaction algorithm
// so pull it out once now.
var roomVersion string
if len(redactTheseEventIDs) > 0 {
createEventJSON, err := a.eventsTable.SelectCreateEvent(txn, roomID)
if err != nil {
return 0, nil, fmt.Errorf("SelectCreateEvent: %s", err)
kegsay marked this conversation as resolved.
Show resolved Hide resolved
}
roomVersion = gjson.GetBytes(createEventJSON, "content.room_version").Str
if roomVersion == "" {
// Defaults to "1" if the key does not exist.
roomVersion = "1"
}
if err = a.eventsTable.Redact(txn, roomVersion, redactTheseEventIDs); err != nil {
return 0, nil, err
}
kegsay marked this conversation as resolved.
Show resolved Hide resolved
}

for _, ev := range newEvents {
var replacesNID int64
// the snapshot ID we assign to this event is unaffected by whether /this/ event is state or not,
Expand Down
33 changes: 33 additions & 0 deletions state/event_table.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package state

import (
"database/sql"
"encoding/json"
"fmt"
"math"

Expand All @@ -10,6 +11,7 @@ import (
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"

"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/sliding-sync/internal"
"github.com/matrix-org/sliding-sync/sqlutil"
)
Expand Down Expand Up @@ -336,6 +338,30 @@ func (t *EventTable) LatestEventNIDInRooms(txn *sqlx.Tx, roomIDs []string, highe
return
}

func (t *EventTable) Redact(txn *sqlx.Tx, roomVer string, eventIDs []string) error {
// verifyAll=false so if we are asked to redact an event we don't have we don't fall over.
eventsToRedact, err := t.SelectByIDs(txn, false, eventIDs)
if err != nil {
return fmt.Errorf("EventTable.Redact[%v]: %s", eventIDs, err)
kegsay marked this conversation as resolved.
Show resolved Hide resolved
}
rv, err := gomatrixserverlib.GetRoomVersion(gomatrixserverlib.RoomVersion(roomVer))
if err != nil {
// unknown room version... let's just default to "1"
kegsay marked this conversation as resolved.
Show resolved Hide resolved
rv = gomatrixserverlib.MustGetRoomVersion(gomatrixserverlib.RoomVersionV1)
}
for i := range eventsToRedact {
eventsToRedact[i].JSON, err = rv.RedactEventJSON(eventsToRedact[i].JSON)
if err != nil {
return fmt.Errorf("RedactEventJSON[%s]: %s", eventsToRedact[i].ID, err)
}
_, err = txn.Exec(`UPDATE syncv3_events SET event=$1 WHERE event_id=$2`, eventsToRedact[i].JSON, eventsToRedact[i].ID)
if err != nil {
return fmt.Errorf("cannot update event %s: %v", eventsToRedact[i].ID, err)
}
}
return nil
}

func (t *EventTable) SelectLatestEventsBetween(txn *sqlx.Tx, roomID string, lowerExclusive, upperInclusive int64, limit int) ([]Event, error) {
var events []Event
// do not pull in events which were in the v2 state block
Expand Down Expand Up @@ -440,6 +466,13 @@ func (t *EventTable) SelectClosestPrevBatch(txn *sqlx.Tx, roomID string, eventNI
return
}

func (t *EventTable) SelectCreateEvent(txn *sqlx.Tx, roomID string) (json.RawMessage, error) {
var evJSON []byte
// there is only 1 create event
err := txn.QueryRow(`SELECT event FROM syncv3_events WHERE room_id=$1 AND event_type='m.room.create' AND state_key=''`, roomID).Scan(&evJSON)
return evJSON, err
}

type EventChunker []Event

func (c EventChunker) Len() int {
Expand Down
166 changes: 166 additions & 0 deletions state/event_table_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package state
import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
"reflect"
"testing"
Expand Down Expand Up @@ -924,3 +925,168 @@ func TestEventTableSelectUnknownEventIDs(t *testing.T) {
}
}
}

func TestEventTableRedact(t *testing.T) {
db, close := connectToDB(t)
defer close()
table := NewEventTable(db)
txn, err := db.Beginx()
if err != nil {
t.Fatalf("failed to start txn: %s", err)
}
defer txn.Rollback()

testCases := []struct {
original map[string]interface{}
roomVer string
wantContent map[string]interface{}
}{
// sanity check
{
original: map[string]interface{}{
"type": "m.room.message",
"sender": "@nasty-mc-nastyface:localhost",
"content": map[string]interface{}{
"msgtype": "m.text",
"body": "Something not very nice",
},
},
wantContent: map[string]interface{}{},
roomVer: "10",
kegsay marked this conversation as resolved.
Show resolved Hide resolved
},
// keep membership
{
original: map[string]interface{}{
"type": "m.room.member",
"state_key": "@nasty-mc-nastyface:localhost",
"sender": "@nasty-mc-nastyface:localhost",
"content": map[string]interface{}{
"membership": "join",
"displayname": "Something not very nice",
"avatar_url": "mxc://something/not/very/nice",
},
},
wantContent: map[string]interface{}{
"membership": "join",
},
roomVer: "10",
},
// keep join rule
{
original: map[string]interface{}{
"type": "m.room.join_rules",
"state_key": "",
"sender": "@nasty-mc-nastyface:localhost",
"content": map[string]interface{}{
"join_rule": "invite",
"extra_data": "not very nice",
},
},
wantContent: map[string]interface{}{
"join_rule": "invite",
},
roomVer: "10",
},
// keep his vis
{
original: map[string]interface{}{
"type": "m.room.history_visibility",
"state_key": "",
"sender": "@nasty-mc-nastyface:localhost",
"content": map[string]interface{}{
"history_visibility": "shared",
"extra_data": "not very nice",
},
},
wantContent: map[string]interface{}{
"history_visibility": "shared",
},
roomVer: "10",
},
// keep version specific fields
{
original: map[string]interface{}{
"type": "m.room.member",
"state_key": "@nasty-mc-nastyface:localhost",
"sender": "@nasty-mc-nastyface:localhost",
"content": map[string]interface{}{
"membership": "join",
"displayname": "Something not very nice",
"avatar_url": "mxc://something/not/very/nice",
"join_authorised_via_users_server": "@bob:localhost",
},
},
wantContent: map[string]interface{}{
"membership": "join",
"join_authorised_via_users_server": "@bob:localhost",
},
roomVer: "10",
},
// remove same field on lower room version
{
original: map[string]interface{}{
"type": "m.room.member",
"state_key": "@nasty-mc-nastyface:localhost",
"sender": "@nasty-mc-nastyface:localhost",
"content": map[string]interface{}{
"membership": "join",
"displayname": "Something not very nice",
"avatar_url": "mxc://something/not/very/nice",
"join_authorised_via_users_server": "@bob:localhost",
},
},
wantContent: map[string]interface{}{
"membership": "join",
},
roomVer: "2",
},
}

roomID := "!TestEventTableRedact"
for i, tc := range testCases {
eventID := fmt.Sprintf("$TestEventTableRedact_%d", i)
tc.original["event_id"] = eventID
tc.original["room_id"] = roomID
stateKey := ""
if tc.original["state_key"] != nil {
stateKey = tc.original["state_key"].(string)
}
j, err := json.Marshal(tc.original)
assertNoError(t, err)
table.Insert(txn, []Event{
{
ID: eventID,
Type: tc.original["type"].(string),
RoomID: roomID,
StateKey: stateKey,
BeforeStateSnapshotID: 22,
JSON: j,
},
}, false)
assertNoError(t, table.Redact(txn, tc.roomVer, []string{eventID}))
gots, err := table.SelectByIDs(txn, true, []string{eventID})
assertNoError(t, err)
if len(gots) != 1 {
t.Errorf("SelectByIDs: got %v results, want 1", len(gots))
continue
}
got := gots[0]
assertVal(t, "event id mismatch", got.ID, eventID)
assertVal(t, "room id mismatch", got.RoomID, roomID)
var gotJSON map[string]interface{}
assertNoError(t, json.Unmarshal(got.JSON, &gotJSON))
assertVal(t, "content mismatch", gotJSON["content"], tc.wantContent)
}
}

func TestEventTableRedactMissingOK(t *testing.T) {
db, close := connectToDB(t)
defer close()
table := NewEventTable(db)
txn, err := db.Beginx()
if err != nil {
t.Fatalf("failed to start txn: %s", err)
}
defer txn.Rollback()
assertNoError(t, table.Redact(txn, "2", []string{"$unknown", "$event", "$ids"}))
}
4 changes: 2 additions & 2 deletions state/storage_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"time"

"github.com/jmoiron/sqlx"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/gomatrixserverlib/spec"
"github.com/matrix-org/sliding-sync/internal"
"github.com/matrix-org/sliding-sync/sqlutil"
"github.com/matrix-org/sliding-sync/testutils"
Expand Down Expand Up @@ -308,7 +308,7 @@ func TestVisibleEventNIDsBetween(t *testing.T) {
t.Fatalf("LatestEventNID: %s", err)
}

baseTimestamp := gomatrixserverlib.Timestamp(1632131678061).Time()
baseTimestamp := spec.Timestamp(1632131678061).Time()
// Test the examples
// Stream Positions
// 1 2 3 4 5 6 7 8 9 10
Expand Down
3 changes: 1 addition & 2 deletions sync2/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"net/url"
"time"

"github.com/matrix-org/gomatrixserverlib"
"github.com/tidwall/gjson"
)

Expand Down Expand Up @@ -135,7 +134,7 @@ type SyncResponse struct {
NextBatch string `json:"next_batch"`
AccountData EventsResponse `json:"account_data"`
Presence struct {
Events []gomatrixserverlib.ClientEvent `json:"events,omitempty"`
Events []json.RawMessage `json:"events,omitempty"`
} `json:"presence"`
Rooms SyncRoomsResponse `json:"rooms"`
ToDevice EventsResponse `json:"to_device"`
Expand Down
Loading