Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

SHHS - Room Join Complexity #5072

Merged
merged 50 commits into from
May 20, 2019
Merged
Changes from 17 commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
ab904c1
temp
hawkowl Apr 16, 2019
cac8907
initial
hawkowl Apr 16, 2019
b381f0f
test
hawkowl Apr 17, 2019
e22e7fa
Merge remote-tracking branch 'origin/develop' into hawkowl/shhs-join-…
hawkowl Apr 17, 2019
269166a
cleanup
hawkowl Apr 17, 2019
aa19990
changelog
hawkowl Apr 17, 2019
b04a635
move to the federation server
hawkowl Apr 17, 2019
817f28a
Remove Python 2.7 testing, to speed things up on SHHS.
hawkowl Apr 17, 2019
04525e9
fixes
hawkowl Apr 17, 2019
cdd5cf0
fixes
hawkowl Apr 17, 2019
10bf9de
tests + preflight check
hawkowl Apr 23, 2019
08180cc
isort
hawkowl Apr 23, 2019
6305ddb
implement post-leave
hawkowl Apr 24, 2019
d436868
fix builds
hawkowl Apr 24, 2019
1cb467c
fix
hawkowl Apr 24, 2019
4c93938
py2 backport division
hawkowl Apr 24, 2019
b9d9a98
Merge remote-tracking branch 'origin/shhs' into hawkowl/shhs-join-com…
hawkowl Apr 25, 2019
4687115
Merge remote-tracking branch 'origin/develop' into hawkowl/shhs-join-…
hawkowl May 8, 2019
ee8605d
Merge remote-tracking branch 'origin/develop' into hawkowl/shhs-join-…
hawkowl May 8, 2019
cc35f21
review comment
hawkowl May 8, 2019
92f71d4
review comment
hawkowl May 8, 2019
580b01f
Merge remote-tracking branch 'origin/shhs' into hawkowl/shhs-join-com…
hawkowl May 8, 2019
2a1e5da
fix
hawkowl May 8, 2019
bd18134
fix
hawkowl May 10, 2019
b8365a6
Merge remote-tracking branch 'origin/develop' into hawkowl/test-confi…
hawkowl May 10, 2019
cafe839
fixes, porting
hawkowl May 10, 2019
d963a11
fixes, porting
hawkowl May 10, 2019
ec3d07c
changelog
hawkowl May 10, 2019
1ba0ecc
finish porting
hawkowl May 10, 2019
338bc44
Merge remote-tracking branch 'origin/develop' into hawkowl/shhs-join-…
hawkowl May 13, 2019
1c9b773
Merge branch 'hawkowl/test-config-parse' into hawkowl/shhs-join-compl…
hawkowl May 13, 2019
e9df0df
Merge remote-tracking branch 'origin/shhs' into hawkowl/shhs-join-com…
hawkowl May 13, 2019
d86ea48
update ratelimiting
hawkowl May 13, 2019
b099b86
remove old uses of the config obj
hawkowl May 13, 2019
451fbf0
remove the federation rc from tests
hawkowl May 13, 2019
d74dfa2
fix
hawkowl May 13, 2019
dac602c
fix
hawkowl May 13, 2019
ce5d662
black
hawkowl May 13, 2019
9eb2cac
Merge branch 'hawkowl/ratelimit-consistency' into hawkowl/shhs-join-c…
hawkowl May 14, 2019
41c33a0
fix
hawkowl May 14, 2019
e07c6b9
fix
hawkowl May 15, 2019
f7f8150
split out, clean up
hawkowl May 15, 2019
2c31a96
cleanup
hawkowl May 15, 2019
b9433d7
Merge remote-tracking branch 'origin/shhs' into hawkowl/shhs-join-com…
hawkowl May 16, 2019
906a13a
cleanup
hawkowl May 16, 2019
e24d09d
Merge remote-tracking branch 'origin/develop' into hawkowl/shhs-join-…
hawkowl May 20, 2019
a6fd6e4
review fixes
hawkowl May 20, 2019
1994d2a
fix
hawkowl May 20, 2019
41c224b
Merge remote-tracking branch 'origin/shhs' into hawkowl/shhs-join-com…
hawkowl May 20, 2019
27021cd
Update pyproject.toml
hawkowl May 20, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -37,4 +37,4 @@ _trial_temp*/
/docs/build/
/htmlcov
/pip-wheel-metadata/

logs/
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
1 change: 1 addition & 0 deletions changelog.d/5072.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events). This option can be used to prevent adverse performance on resource-constrained homeservers.
7 changes: 7 additions & 0 deletions docs/sample_config.yaml
Original file line number Diff line number Diff line change
@@ -239,6 +239,13 @@ listeners:
# Used by phonehome stats to group together related servers.
#server_context: context

# Resource-constrained Homeserver Settings
#
# Requires _joins to be set to True.
#
#limit_large_remote_room_joins: False
#limit_large_remote_room_complexity: 1.0


## TLS ##

3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
[build-system]
requires = ["setuptools", "wheel"]
hawkowl marked this conversation as resolved.
Show resolved Hide resolved

[tool.towncrier]
package = "synapse"
filename = "CHANGES.md"
1 change: 1 addition & 0 deletions synapse/api/urls.py
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@
FEDERATION_PREFIX = "/_matrix/federation"
FEDERATION_V1_PREFIX = FEDERATION_PREFIX + "/v1"
FEDERATION_V2_PREFIX = FEDERATION_PREFIX + "/v2"
FEDERATION_UNSTABLE_PREFIX = FEDERATION_PREFIX + "/unstable"
STATIC_PREFIX = "/_matrix/static"
WEB_CLIENT_PREFIX = "/_matrix/client"
CONTENT_REPO_PREFIX = "/_matrix/content"
13 changes: 13 additions & 0 deletions synapse/config/server.py
Original file line number Diff line number Diff line change
@@ -172,6 +172,12 @@ def read_config(self, config):

self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))

# Resource-constrained Homeserver Configuration
self.limit_large_room_joins = config.get("limit_large_remote_room_joins", False)
self.limit_large_room_complexity = config.get(
"limit_large_remote_room_complexity", 1.0
)

bind_port = config.get("bind_port")
if bind_port:
if config.get("no_tls", False):
@@ -490,6 +496,13 @@ def default_config(self, server_name, data_dir_path, **kwargs):

# Used by phonehome stats to group together related servers.
#server_context: context

# Resource-constrained Homeserver Settings
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
#
# Requires _joins to be set to True.
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
#
#limit_large_remote_room_joins: False
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
#limit_large_remote_room_complexity: 1.0
""" % locals()

def read_arguments(self, args):
33 changes: 33 additions & 0 deletions synapse/federation/federation_client.py
Original file line number Diff line number Diff line change
@@ -992,3 +992,36 @@ def forward_third_party_invite(self, destinations, room_id, event_dict):
)

raise RuntimeError("Failed to send to any server.")

@defer.inlineCallbacks
def get_room_complexity(self, destination, room_id):
"""
Fetch the complexity of a remote room from another server.

Args:
destination (str): The remote server
room_id (str): The room ID to ask about.

Returns:
Deferred[dict] or Deferred[None]: Dict contains the complexity
metric versions, while None means we could not fetch the complexity.
"""
try:
complexity = yield self.transport_layer.get_room_complexity(
destination=destination,
room_id=room_id
)
defer.returnValue(complexity)
except CodeMessageException:
# We didn't manage to get it -- probably a 404. We are okay if other
# servers don't give it to us.
pass
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
except Exception as e:
logger.exception(
"Failed to fetch room complexity via %s for %s: %s",
destination, room_id, str(e)
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
)

# If we don't manage to find it, return None. It's not an error if a
# server doesn't give it to us.
defer.returnValue(None)
38 changes: 29 additions & 9 deletions synapse/federation/transport/client.py
Original file line number Diff line number Diff line change
@@ -21,7 +21,11 @@
from twisted.internet import defer

from synapse.api.constants import Membership
from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
from synapse.api.urls import (
FEDERATION_UNSTABLE_PREFIX,
FEDERATION_V1_PREFIX,
FEDERATION_V2_PREFIX,
)
from synapse.util.logutils import log_function

logger = logging.getLogger(__name__)
@@ -959,6 +963,28 @@ def bulk_get_publicised_groups(self, destination, user_ids):
ignore_backoff=True,
)

def get_room_complexity(self, destination, room_id):
"""
Args:
destination (str): The remote server
room_id (str): The room ID to ask about.
"""
path = _create_path(
FEDERATION_UNSTABLE_PREFIX, "/rooms/%s/complexity", room_id
)

return self.client.get_json(
destination=destination,
path=path
)


def _create_path(federation_prefix, path, *args):
"""
Ensures that all args are url encoded.
"""
return federation_prefix + path % tuple(urllib.parse.quote(arg, "") for arg in args)


def _create_v1_path(path, *args):
"""Creates a path against V1 federation API from the path template and
@@ -975,10 +1001,7 @@ def _create_v1_path(path, *args):
Returns:
str
"""
return (
FEDERATION_V1_PREFIX
+ path % tuple(urllib.parse.quote(arg, "") for arg in args)
)
return _create_path(FEDERATION_V1_PREFIX, path, *args)


def _create_v2_path(path, *args):
@@ -996,7 +1019,4 @@ def _create_v2_path(path, *args):
Returns:
str
"""
return (
FEDERATION_V2_PREFIX
+ path % tuple(urllib.parse.quote(arg, "") for arg in args)
)
return _create_path(FEDERATION_V2_PREFIX, path, *args)
27 changes: 26 additions & 1 deletion synapse/federation/transport/server.py
Original file line number Diff line number Diff line change
@@ -23,7 +23,11 @@
import synapse
from synapse.api.errors import Codes, FederationDeniedError, SynapseError
from synapse.api.room_versions import RoomVersions
from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
from synapse.api.urls import (
FEDERATION_UNSTABLE_PREFIX,
FEDERATION_V1_PREFIX,
FEDERATION_V2_PREFIX,
)
from synapse.http.endpoint import parse_and_validate_server_name
from synapse.http.server import JsonResource
from synapse.http.servlet import (
@@ -1299,6 +1303,26 @@ def on_PUT(self, origin, content, query, group_id):
defer.returnValue((200, new_content))


class RoomComplexityServlet(BaseFederationServlet):
PATH = "/rooms/(?P<room_id>[^/]*)/complexity"
PREFIX = FEDERATION_UNSTABLE_PREFIX

@defer.inlineCallbacks
def on_GET(self, origin, content, query, room_id):

store = self.handler.hs.get_datastore()

is_public = yield store.is_room_world_readable_or_publicly_joinable(
room_id
)

if not is_public:
raise SynapseError(404, "Room not found", errcode=Codes.INVALID_PARAM)
hawkowl marked this conversation as resolved.
Show resolved Hide resolved

complexity = yield store.get_room_complexity(room_id)
defer.returnValue((200, complexity))


FEDERATION_SERVLET_CLASSES = (
FederationSendServlet,
FederationEventServlet,
@@ -1322,6 +1346,7 @@ def on_PUT(self, origin, content, query, group_id):
FederationThirdPartyInviteExchangeServlet,
On3pidBindServlet,
FederationVersionServlet,
RoomComplexityServlet,
)

OPENID_SERVLET_CLASSES = (
25 changes: 25 additions & 0 deletions synapse/handlers/federation.py
Original file line number Diff line number Diff line change
@@ -2724,3 +2724,28 @@ def user_joined_room(self, user, room_id):
)
else:
return user_joined_room(self.distributor, user, room_id)

@defer.inlineCallbacks
def get_room_complexity(self, remote_room_hosts, room_id):
"""
Fetch the complexity of a remote room over federation.

Args:
remote_room_hosts (list[str]): The remote servers to ask.
room_id (str): The room ID to ask about.

Returns:
Deferred[dict] or Deferred[None]: Dict contains the complexity
metric versions, while None means we could not fetch the complexity.
"""

for host in remote_room_hosts:
res = yield self.federation_client.get_room_complexity(host, room_id)

# We got a result, return it.
if res:
defer.returnValue(res)

# We fell off the bottom, couldn't get the complexity from anyone. Oh
# well.
defer.returnValue(None)
52 changes: 48 additions & 4 deletions synapse/handlers/room_member.py
Original file line number Diff line number Diff line change
@@ -25,8 +25,7 @@

from twisted.internet import defer

import synapse.server
import synapse.types
from synapse import types
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
from synapse.api.constants import EventTypes, Membership
from synapse.api.errors import AuthError, Codes, SynapseError
from synapse.types import RoomID, UserID
@@ -574,7 +573,7 @@ def send_membership_event(
)
assert self.hs.is_mine(sender), "Sender must be our own: %s" % (sender,)
else:
requester = synapse.types.create_requester(target_user)
requester = types.create_requester(target_user)

prev_event = yield self.event_creation_handler.deduplicate_state_event(
event, context,
@@ -1004,14 +1003,33 @@ def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
# filter ourselves out of remote_room_hosts: do_invite_join ignores it
# and if it is the only entry we'd like to return a 404 rather than a
# 500.

remote_room_hosts = [
host for host in remote_room_hosts if host != self.hs.hostname
]

if len(remote_room_hosts) == 0:
raise SynapseError(404, "No known servers")

if self.hs.config.limit_large_room_joins:
# Go fetch the room complexity here...
complexity_fetched = False
complexity = yield self.federation_handler.get_room_complexity(
remote_room_hosts, room_id
)

max_complexity = self.hs.config.limit_large_room_joins_complexity

if complexity:
if complexity["v1"] > max_complexity:
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
msg = "Room too large (preflight) -- %d > %d" % (
complexity["v1"], max_complexity
)
raise SynapseError(
code=400, msg=msg,
errcode=Codes.RESOURCE_LIMIT_EXCEEDED
)
complexity_fetched = True

# We don't do an auth check if we are doing an invite
# join dance for now, since we're kinda implicitly checking
# that we are allowed to join when we decide whether or not we
@@ -1024,6 +1042,32 @@ def _remote_join(self, requester, remote_room_hosts, room_id, user, content):
)
yield self._user_joined_room(user, room_id)

# Check the room we just joined wasn't too large, if we didn't fetch the
# complexity of it before.
if self.hs.config.limit_large_room_joins:
if not complexity_fetched:
# We don't know the room complexity, so let's take a look.
hawkowl marked this conversation as resolved.
Show resolved Hide resolved
complexity = yield self.store.get_room_complexity(room_id)

if complexity["v1"] > max_complexity:
requester = types.create_requester(
user, None, False, None
)

yield self.update_membership(
requester=requester,
target=user,
room_id=room_id,
action="leave"
)
msg = "Room too large (postflight) -- %d > %d" % (
complexity["v1"], max_complexity
)
raise SynapseError(
code=400, msg=msg,
errcode=Codes.RESOURCE_LIMIT_EXCEEDED
)

@defer.inlineCallbacks
def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
"""Implements RoomMemberHandler._remote_reject_invite
44 changes: 44 additions & 0 deletions synapse/storage/events_worker.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import division

import itertools
import logging
from collections import namedtuple
@@ -591,3 +593,45 @@ def f(txn):
return res

return self.runInteraction("get_rejection_reasons", f)

def _get_state_event_counts_txn(self, txn, room_id):
"""
See get_state_event_counts.
"""
sql = "SELECT COUNT(*) FROM state_events WHERE room_id=?"
txn.execute(sql, (room_id,))
row = txn.fetchone()
return row[0] if row else 0

def get_state_event_counts(self, room_id):
"""
Gets the total number of state events in a room.

Args:
room_id (str)

Returns:
Deferred[int]
"""
return self.runInteraction(
"get_state_event_counts", self._get_state_event_counts_txn, room_id
)

@defer.inlineCallbacks
def get_room_complexity(self, room_id):
"""
Get the complexity of a room.

Args:
room_id (str)

Returns:
Deferred[dict[str:int]] of complexity version to complexity.
"""
state_events = yield self.get_state_event_counts(room_id)

# Call this one "v1", so we can introduce new ones as we want to develop
# it.
complexity_v1 = round(state_events / 500, 2)

defer.returnValue({"v1": complexity_v1})
Loading