Merge pull request #98 from matrix-org/dmr/migration

Migrate existing data to new schema on startup

Dockerfile

@@ -3,11 +3,12 @@ FROM docker.io/golang:1.19-alpine AS base
 WORKDIR /build
 
 RUN apk --update --no-cache add build-base git
+ARG BINARYNAME=syncv3
 RUN --mount=target=. \
     --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg/mod \
   GIT_COMMIT=$(git rev-list -1 HEAD) && \
-  go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -trimpath -o /out/ ./cmd/syncv3
+  go build -ldflags "-X main.GitCommit=$GIT_COMMIT" -trimpath -o /out/syncv3 "./cmd/$BINARYNAME"
 
 FROM alpine:3.17
 
@@ -18,4 +19,7 @@ ENV SYNCV3_BINDADDR="0.0.0.0:8008"
 EXPOSE 8008
 
 WORKDIR /usr/bin
+# It would be nice if the binary we exec was called $BINARYNAME here, but build args
+# aren't expanded in ENTRYPOINT directives. Instead, we always call the output binary
+# "syncv3". (See https://github.com/moby/moby/issues/18492)
 ENTRYPOINT /usr/bin/syncv3

cmd/syncv3/main.go

@@ -131,6 +131,11 @@ func main() {
 		}
 	}
 
+	err := sync2.MigrateDeviceIDs(args[EnvServer], args[EnvDB], args[EnvSecret], true)
+	if err != nil {
+		panic(err)
+	}
+
 	h2, h3 := syncv3.Setup(args[EnvServer], args[EnvDB], args[EnvSecret], syncv3.Opts{
 		Debug:                args[EnvDebug] == "1",
 		AddPrometheusMetrics: args[EnvPrometheus] != "",

cmd/testmigration/main.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"github.com/matrix-org/sliding-sync/sync2"
+	"os"
+)
+
+const (
+	// Required fields
+	EnvServer = "SYNCV3_SERVER"
+	EnvDB     = "SYNCV3_DB"
+	EnvSecret = "SYNCV3_SECRET"
+
+	// Migration test only
+	EnvMigrationCommit = "SYNCV3_TEST_MIGRATION_COMMIT"
+)
+
+func main() {
+	args := map[string]string{
+		EnvServer:          os.Getenv(EnvServer),
+		EnvDB:              os.Getenv(EnvDB),
+		EnvSecret:          os.Getenv(EnvSecret),
+		EnvMigrationCommit: os.Getenv(EnvMigrationCommit),
+	}
+
+	err := sync2.MigrateDeviceIDs(args[EnvServer], args[EnvDB], args[EnvSecret], args[EnvMigrationCommit] != "")
+	if err != nil {
+		panic(err)
+	}
+}

state/to_device_table.go

@@ -81,7 +81,12 @@ func (t *ToDeviceTable) DeleteMessagesUpToAndIncluding(userID, deviceID string,
 }
 
 func (t *ToDeviceTable) DeleteAllMessagesForDevice(userID, deviceID string) error {
+	// TODO: should these deletes take place in a transaction?
 	_, err := t.db.Exec(`DELETE FROM syncv3_to_device_messages WHERE user_id = $1 AND device_id = $2`, userID, deviceID)
+	if err != nil {
+		return err
+	}
+	_, err = t.db.Exec(`DELETE FROM syncv3_to_device_ack_pos WHERE user_id = $1 AND device_id = $2`, userID, deviceID)
 	return err
 }
 

state/txn_table.go

@@ -23,7 +23,7 @@ func NewTransactionsTable(db *sqlx.DB) *TransactionsTable {
 	// make sure tables are made
 	db.MustExec(`
 	CREATE TABLE IF NOT EXISTS syncv3_txns (
-		user_id TEXT NOT NULL, -- was actually device_id before migration
+		user_id TEXT NOT NULL,
 		device_id TEXT NOT NULL,
 		event_id TEXT NOT NULL,
 		txn_id TEXT NOT NULL,