From c1cfd6c20abaac115acd1a4b7cdfbd6974eb506d Mon Sep 17 00:00:00 2001 From: pik Date: Thu, 16 Mar 2017 14:11:00 -0300 Subject: [PATCH 1/7] Change Index on user + filters to Unique * especially relevant since the current increment code is not atomic --- synapse/storage/prepare_database.py | 2 +- .../delta/42/unique_user_filter_index.py | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/42/unique_user_filter_index.py diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index ed84db6b4b49..c38c2e1ed231 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 41 +SCHEMA_VERSION = 42 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/42/unique_user_filter_index.py b/synapse/storage/schema/delta/42/unique_user_filter_index.py new file mode 100644 index 000000000000..244ca619584b --- /dev/null +++ b/synapse/storage/schema/delta/42/unique_user_filter_index.py @@ -0,0 +1,39 @@ +from synapse.storage.engines import PostgresEngine + +import logging + +logger = logging.getLogger(__name__) + + +def run_create(cur, database_engine, *args, **kwargs): + if isinstance(database_engine, PostgresEngine): + select_clause = ''' + CREATE TABLE user_filters_migration AS + SELECT DISTINCT ON (user_id, filter_id) user_id, filter_id, filter_json + FROM user_filters; + ''' + else: + select_clause = ''' + CREATE TABLE user_filters_migration AS + SELECT * FROM user_filters GROUP BY user_id, filter_id; + ''' + sql = ''' + BEGIN; + %s + DROP INDEX user_filters_by_user_id_filter_id; + DELETE FROM user_filters; + INSERT INTO user_filters(user_id, filter_id, filter_json) + SELECT * FROM user_filters_migration; + DROP TABLE user_filters_migration; + CREATE UNIQUE INDEX user_filters_by_user_id_filter_id_unique + ON user_filters(user_id, filter_id); + END; + ''' % select_clause + if isinstance(database_engine, PostgresEngine): + cur.execute(sql) + else: + cur.executescript(sql) + + +def run_upgrade(cur, database_engine, *args, **kwargs): + pass From 9564b2bf86b05007238b15eabc3dae85f9b68b6a Mon Sep 17 00:00:00 2001 From: pik Date: Thu, 16 Mar 2017 14:40:49 -0300 Subject: [PATCH 2/7] Create user_filters_by_user_id_unique index by default --- synapse/storage/schema/delta/42/unique_user_filter_index.py | 4 ++-- synapse/storage/schema/full_schemas/16/push.sql | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/synapse/storage/schema/delta/42/unique_user_filter_index.py b/synapse/storage/schema/delta/42/unique_user_filter_index.py index 244ca619584b..709d115dee1e 100644 --- a/synapse/storage/schema/delta/42/unique_user_filter_index.py +++ b/synapse/storage/schema/delta/42/unique_user_filter_index.py @@ -5,7 +5,7 @@ logger = logging.getLogger(__name__) -def run_create(cur, database_engine, *args, **kwargs): +def run_upgrade(cur, database_engine, *args, **kwargs): if isinstance(database_engine, PostgresEngine): select_clause = ''' CREATE TABLE user_filters_migration AS @@ -35,5 +35,5 @@ def run_create(cur, database_engine, *args, **kwargs): cur.executescript(sql) -def run_upgrade(cur, database_engine, *args, **kwargs): +def run_create(cur, database_engine, *args, **kwargs): pass diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index e44465cf45bd..13562149b72e 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -59,9 +59,8 @@ CREATE TABLE IF NOT EXISTS user_filters( filter_json bytea ); -CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( - user_id, filter_id -); +CREATE UNIQUE INDEX user_filters_by_user_id_filter_id_unique + ON user_filters(user_id, filter_id); CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, From 1336115e05d01184e43a1c5e7fc09270873651d5 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 4 Oct 2019 09:48:41 +0100 Subject: [PATCH 3/7] No need to update original schema --- synapse/storage/schema/full_schemas/16/push.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql index 13562149b72e..e44465cf45bd 100644 --- a/synapse/storage/schema/full_schemas/16/push.sql +++ b/synapse/storage/schema/full_schemas/16/push.sql @@ -59,8 +59,9 @@ CREATE TABLE IF NOT EXISTS user_filters( filter_json bytea ); -CREATE UNIQUE INDEX user_filters_by_user_id_filter_id_unique - ON user_filters(user_id, filter_id); +CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( + user_id, filter_id +); CREATE TABLE IF NOT EXISTS push_rules_enable ( id BIGINT PRIMARY KEY, From 6f7ce598661cc0a97ea36dacda25617d18427d61 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 4 Oct 2019 09:48:59 +0100 Subject: [PATCH 4/7] lint --- .../delta/56/unique_user_filter_index.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/synapse/storage/schema/delta/56/unique_user_filter_index.py b/synapse/storage/schema/delta/56/unique_user_filter_index.py index 709d115dee1e..d3f63350d2c8 100644 --- a/synapse/storage/schema/delta/56/unique_user_filter_index.py +++ b/synapse/storage/schema/delta/56/unique_user_filter_index.py @@ -1,23 +1,24 @@ -from synapse.storage.engines import PostgresEngine - import logging +from synapse.storage.engines import PostgresEngine + logger = logging.getLogger(__name__) def run_upgrade(cur, database_engine, *args, **kwargs): if isinstance(database_engine, PostgresEngine): - select_clause = ''' + select_clause = """ CREATE TABLE user_filters_migration AS SELECT DISTINCT ON (user_id, filter_id) user_id, filter_id, filter_json FROM user_filters; - ''' + """ else: - select_clause = ''' + select_clause = """ CREATE TABLE user_filters_migration AS SELECT * FROM user_filters GROUP BY user_id, filter_id; - ''' - sql = ''' + """ + sql = ( + """ BEGIN; %s DROP INDEX user_filters_by_user_id_filter_id; @@ -28,7 +29,9 @@ def run_upgrade(cur, database_engine, *args, **kwargs): CREATE UNIQUE INDEX user_filters_by_user_id_filter_id_unique ON user_filters(user_id, filter_id); END; - ''' % select_clause + """ + % select_clause + ) if isinstance(database_engine, PostgresEngine): cur.execute(sql) else: From a44e4e06222667f8e804614ae9161b46e5528f26 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 4 Oct 2019 09:57:36 +0100 Subject: [PATCH 5/7] use a temporary table make sure it doesn't get left around --- synapse/storage/schema/delta/56/unique_user_filter_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/delta/56/unique_user_filter_index.py b/synapse/storage/schema/delta/56/unique_user_filter_index.py index d3f63350d2c8..2e9cd09b0351 100644 --- a/synapse/storage/schema/delta/56/unique_user_filter_index.py +++ b/synapse/storage/schema/delta/56/unique_user_filter_index.py @@ -8,13 +8,13 @@ def run_upgrade(cur, database_engine, *args, **kwargs): if isinstance(database_engine, PostgresEngine): select_clause = """ - CREATE TABLE user_filters_migration AS + CREATE TEMPORARY TABLE user_filters_migration AS SELECT DISTINCT ON (user_id, filter_id) user_id, filter_id, filter_json FROM user_filters; """ else: select_clause = """ - CREATE TABLE user_filters_migration AS + CREATE TEMPORARY TABLE user_filters_migration AS SELECT * FROM user_filters GROUP BY user_id, filter_id; """ sql = ( From 7ca31f13b58204e6db37fb3524e36ad62257b702 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 4 Oct 2019 09:57:57 +0100 Subject: [PATCH 6/7] Let's make the columns non-null while we're here --- synapse/storage/schema/delta/56/unique_user_filter_index.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/storage/schema/delta/56/unique_user_filter_index.py b/synapse/storage/schema/delta/56/unique_user_filter_index.py index 2e9cd09b0351..4efc1a586ffe 100644 --- a/synapse/storage/schema/delta/56/unique_user_filter_index.py +++ b/synapse/storage/schema/delta/56/unique_user_filter_index.py @@ -23,6 +23,10 @@ def run_upgrade(cur, database_engine, *args, **kwargs): %s DROP INDEX user_filters_by_user_id_filter_id; DELETE FROM user_filters; + ALTER TABLE user_filters + ALTER COLUMN user_id SET NOT NULL + ALTER COLUMN filter_id SET NOT NULL + ALTER COLUMN filter_json SET NOT NULL; INSERT INTO user_filters(user_id, filter_id, filter_json) SELECT * FROM user_filters_migration; DROP TABLE user_filters_migration; From 850e0a07ec30bbe8ee1f12d128266207c684f25b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 4 Oct 2019 10:00:50 +0100 Subject: [PATCH 7/7] newsfile --- changelog.d/1172.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1172.misc diff --git a/changelog.d/1172.misc b/changelog.d/1172.misc new file mode 100644 index 000000000000..30b3e560825b --- /dev/null +++ b/changelog.d/1172.misc @@ -0,0 +1 @@ +Update `user_filters` table to have a unique index, and non-null columns. Thanks to @pik for contributing this. \ No newline at end of file