From 8daaef078b6a6aef99ad0120038d119a5463e8cc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 May 2022 17:06:22 +0100 Subject: [PATCH 1/5] Add index to cache invalidations For workers that rarely write to the cache the `get_all_updated_caches` query can become expensive if the worker falls behind when reading the cache. --- synapse/storage/databases/main/cache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index dd4e83a2ad19..b3026fbdc558 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -57,6 +57,13 @@ def __init__( self._instance_name = hs.get_instance_name() + self.db_pool.updates.register_background_index_update( + update_name="cache_invalidation_index_by_instance", + index_name="cache_invalidation_stream_by_instance_instance_index", + table="cache_invalidation_stream_by_instance", + columns=("instance_name", "stream_id"), + ) + async def get_all_updated_caches( self, instance_name: str, last_id: int, current_id: int, limit: int ) -> Tuple[List[Tuple[int, tuple]], int, bool]: From 47cac7223829d905b2f05ee80ea887bb33eea506 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 May 2022 17:08:14 +0100 Subject: [PATCH 2/5] Newsfile --- changelog.d/12747.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12747.bugfix diff --git a/changelog.d/12747.bugfix b/changelog.d/12747.bugfix new file mode 100644 index 000000000000..f75d59414e1f --- /dev/null +++ b/changelog.d/12747.bugfix @@ -0,0 +1 @@ +Fix database performance problem for large servers with lots of workers. From b31a15c2ecd44bbbcf0c48fb4a5104da4bcfe1e4 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 16 May 2022 17:12:00 +0100 Subject: [PATCH 3/5] Actually add the delta file... --- .../delta/69/02cache_invalidation_index.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql diff --git a/synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql b/synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql new file mode 100644 index 000000000000..22ae3b8c0005 --- /dev/null +++ b/synapse/storage/schema/main/delta/69/02cache_invalidation_index.sql @@ -0,0 +1,18 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Background update to clear the inboxes of hidden and deleted devices. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (6902, 'cache_invalidation_index_by_instance', '{}'); From da31e41dd458206bfc6e05340d9c3d6e1acad4a1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 May 2022 08:42:08 +0100 Subject: [PATCH 4/5] Update changelog.d/12747.bugfix Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com> --- changelog.d/12747.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/12747.bugfix b/changelog.d/12747.bugfix index f75d59414e1f..0fb0059237cc 100644 --- a/changelog.d/12747.bugfix +++ b/changelog.d/12747.bugfix @@ -1 +1 @@ -Fix database performance problem for large servers with lots of workers. +Fix poor database performance when reading the cache invalidation stream for large servers with lots of workers. From 1104c6c2e0bf6d0b5dd740c482c20abb9c0b26b8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 May 2022 08:42:58 +0100 Subject: [PATCH 5/5] Psql only --- synapse/storage/databases/main/cache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index b3026fbdc558..1653a6a9b694 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -62,6 +62,7 @@ def __init__( index_name="cache_invalidation_stream_by_instance_instance_index", table="cache_invalidation_stream_by_instance", columns=("instance_name", "stream_id"), + psql_only=True, # The table is only on postgres DBs. ) async def get_all_updated_caches(