Sliding Sync: Speed up background updates to populate Sliding Sync tables

changelog.d/17676.misc

@@ -0,0 +1 @@
+Pre-populate room data used in experimental [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575) Sliding Sync `/sync` endpoint for quick filtering/sorting.

synapse/storage/database.py

@@ -1472,6 +1472,11 @@ async def simple_upsert_many(
         key_values: Collection[Collection[Any]],
         value_names: Collection[str],
         value_values: Collection[Collection[Any]],
+        # Given these are the same type as the normal values, force keyword-only so that
+        # they can't be confused.
+        *,
+        insertion_value_names: Collection[str] = [],
+        insertion_value_values: Collection[Iterable[Any]] = [],
         desc: str,
     ) -> None:
         """
@@ -1497,6 +1502,8 @@ async def simple_upsert_many(
             key_values,
             value_names,
             value_values,
+            insertion_value_names=insertion_value_names,
+            insertion_value_values=insertion_value_values,
             db_autocommit=autocommit,
         )
 
@@ -1508,6 +1515,11 @@ def simple_upsert_many_txn(
         key_values: Collection[Iterable[Any]],
         value_names: Collection[str],
         value_values: Collection[Iterable[Any]],
+        # Given these are the same type as the normal values, force keyword-only so that
+        # they can't be confused.
+        *,
+        insertion_value_names: Collection[str] = [],
+        insertion_value_values: Collection[Iterable[Any]] = [],
     ) -> None:
         """
         Upsert, many times.
@@ -1519,6 +1531,9 @@ def simple_upsert_many_txn(
             value_names: The value column names
             value_values: A list of each row's value column values.
                 Ignored if value_names is empty.
+            insertion_value_names: The value column names to use only when inserting
+            insertion_value_values: A list of each row's value column values to use only
+                when inserting. Ignored if `insertion_value_names` is empty.
         """
         # If there's nothing to upsert, then skip executing the query.
         if not key_values:
@@ -1528,14 +1543,30 @@ def simple_upsert_many_txn(
         # zip() works correctly.
         if not value_names:
             value_values = [() for x in range(len(key_values))]
-        elif len(value_values) != len(key_values):
+        elif len(key_values) != len(value_values):
             raise ValueError(
                 f"{len(key_values)} key rows and {len(value_values)} value rows: should be the same number."
             )
 
+        # No value columns, therefore make a blank list so that the following
+        # zip() works correctly.
+        if not insertion_value_names:
+            insertion_value_values = [() for x in range(len(key_values))]
+        elif len(key_values) != len(insertion_value_values):
+            raise ValueError(
+                f"{len(key_values)} key rows and {len(insertion_value_values)} insertion value rows: should be the same number."
+            )
+
         if table not in self._unsafe_to_upsert_tables:
             return self.simple_upsert_many_txn_native_upsert(
-                txn, table, key_names, key_values, value_names, value_values
+                txn,
+                table,
+                key_names,
+                key_values,
+                value_names,
+                value_values,
+                insertion_value_names=insertion_value_names,
+                insertion_value_values=insertion_value_values,
             )
         else:
             return self.simple_upsert_many_txn_emulated(
@@ -1545,6 +1576,8 @@ def simple_upsert_many_txn(
                 key_values,
                 value_names,
                 value_values,
+                insertion_value_names=insertion_value_names,
+                insertion_value_values=insertion_value_values,
             )
 
     def simple_upsert_many_txn_emulated(
@@ -1555,6 +1588,11 @@ def simple_upsert_many_txn_emulated(
         key_values: Collection[Iterable[Any]],
         value_names: Collection[str],
         value_values: Iterable[Iterable[Any]],
+        # Given these are the same type as the normal values, force keyword-only so that
+        # they can't be confused.
+        *,
+        insertion_value_names: Collection[str] = [],
+        insertion_value_values: Collection[Iterable[Any]] = [],
     ) -> None:
         """
         Upsert, many times, but without native UPSERT support or batching.
@@ -1566,18 +1604,26 @@ def simple_upsert_many_txn_emulated(
             value_names: The value column names
             value_values: A list of each row's value column values.
                 Ignored if value_names is empty.
+            insertion_value_names: The value column names to use only when inserting
+            insertion_value_values: A list of each row's value column values to use only
+                when inserting. Ignored if `insertion_value_names` is empty.
         """
 
         # Lock the table just once, to prevent it being done once per row.
         # Note that, according to Postgres' documentation, once obtained,
         # the lock is held for the remainder of the current transaction.
         self.engine.lock_table(txn, table)
 
-        for keyv, valv in zip(key_values, value_values):
+        for keyv, valv, insertionv in zip(
+            key_values, value_values, insertion_value_values
+        ):
             _keys = dict(zip(key_names, keyv))
             _vals = dict(zip(value_names, valv))
+            _insertion_vals = dict(zip(insertion_value_names, insertionv))
 
-            self.simple_upsert_txn_emulated(txn, table, _keys, _vals, lock=False)
+            self.simple_upsert_txn_emulated(
+                txn, table, _keys, _vals, insertion_values=_insertion_vals, lock=False
+            )
 
     @staticmethod
     def simple_upsert_many_txn_native_upsert(
@@ -1587,6 +1633,11 @@ def simple_upsert_many_txn_native_upsert(
         key_values: Collection[Iterable[Any]],
         value_names: Collection[str],
         value_values: Iterable[Iterable[Any]],
+        # Given these are the same type as the normal values, force keyword-only so that
+        # they can't be confused.
+        *,
+        insertion_value_names: Collection[str] = [],
+        insertion_value_values: Collection[Iterable[Any]] = [],
     ) -> None:
         """
         Upsert, many times, using batching where possible.
@@ -1598,10 +1649,14 @@ def simple_upsert_many_txn_native_upsert(
             value_names: The value column names
             value_values: A list of each row's value column values.
                 Ignored if value_names is empty.
+            insertion_value_names: The value column names to use only when inserting
+            insertion_value_values: A list of each row's value column values to use only
+                when inserting. Ignored if `insertion_value_names` is empty.
         """
         allnames: List[str] = []
         allnames.extend(key_names)
         allnames.extend(value_names)
+        allnames.extend(insertion_value_names)
 
         if not value_names:
             latter = "NOTHING"
@@ -1612,8 +1667,8 @@ def simple_upsert_many_txn_native_upsert(
 
         args = []
 
-        for x, y in zip(key_values, value_values):
-            args.append(tuple(x) + tuple(y))
+        for x, y, z in zip(key_values, value_values, insertion_value_values):
+            args.append(tuple(x) + tuple(y) + tuple(z))
 
         if isinstance(txn.database_engine, PostgresEngine):
             # We use `execute_values` as it can be a lot faster than `execute_batch`,

synapse/storage/databases/main/events.py

@@ -137,7 +137,7 @@ class SlidingSyncStateInsertValues(TypedDict, total=False):
     """
 
     room_type: Optional[str]
-    is_encrypted: Optional[bool]
+    is_encrypted: bool
     room_name: Optional[str]
     tombstone_successor_room_id: Optional[str]
 
@@ -150,7 +150,7 @@ class SlidingSyncMembershipSnapshotSharedInsertValues(
     multiple memberships
     """
 
-    has_known_state: Optional[bool]
+    has_known_state: bool
 
 
 @attr.s(slots=True, auto_attribs=True)