From 53ace904b2b8e2444bc518b2498947c869c8c13b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 01:29:25 +0000 Subject: [PATCH 01/45] total WIP skeleton for /room_keys API --- synapse/handlers/e2e_room_keys.py | 60 ++++++++ synapse/rest/client/v2_alpha/room_keys.py | 56 ++++++++ synapse/storage/e2e_room_keys.py | 133 ++++++++++++++++++ .../storage/schema/delta/46/e2e_room_keys.sql | 40 ++++++ 4 files changed, 289 insertions(+) create mode 100644 synapse/handlers/e2e_room_keys.py create mode 100644 synapse/rest/client/v2_alpha/room_keys.py create mode 100644 synapse/storage/e2e_room_keys.py create mode 100644 synapse/storage/schema/delta/46/e2e_room_keys.sql diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py new file mode 100644 index 000000000000..78c838a82950 --- /dev/null +++ b/synapse/handlers/e2e_room_keys.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ujson as json +import logging + +from canonicaljson import encode_canonical_json +from twisted.internet import defer + +from synapse.api.errors import SynapseError, CodeMessageException +from synapse.types import get_domain_from_id +from synapse.util.logcontext import preserve_fn, make_deferred_yieldable +from synapse.util.retryutils import NotRetryingDestination + +logger = logging.getLogger(__name__) + + +class E2eRoomKeysHandler(object): + def __init__(self, hs): + self.store = hs.get_datastore() + + @defer.inlineCallbacks + def get_room_keys(self, user_id, version, room_id, session_id): + results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id) + defer.returnValue(results) + + @defer.inlineCallbacks + def upload_room_keys(self, user_id, version, room_keys): + + # TODO: Validate the JSON to make sure it has the right keys. + + # go through the room_keys + for room_id in room_keys['rooms']: + for session_id in room_keys['rooms'][room_id]['sessions']: + session = room_keys['rooms'][room_id]['sessions'][session_id] + + # get a lock + + # get the room_key for this particular row + yield self.store.get_e2e_room_key() + + # check whether we merge or not + if() + + # if so, we set it + yield self.store.set_e2e_room_key() + + # release the lock diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py new file mode 100644 index 000000000000..9b93001919d0 --- /dev/null +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer + +from synapse.api.errors import SynapseError +from synapse.http.servlet import ( + RestServlet, parse_json_object_from_request, parse_integer +) +from synapse.http.servlet import parse_string +from synapse.types import StreamToken +from ._base import client_v2_patterns + +logger = logging.getLogger(__name__) + + +class RoomKeysUploadServlet(RestServlet): + PATTERNS = client_v2_patterns("/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$") + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(RoomKeysUploadServlet, self).__init__() + self.auth = hs.get_auth() + self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() + + @defer.inlineCallbacks + def on_POST(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=True) + user_id = requester.user.to_string() + body = parse_json_object_from_request(request) + + result = yield self.e2e_room_keys_handler.upload_room_keys( + user_id, version, body + ) + defer.returnValue((200, result)) + + +def register_servlets(hs, http_server): + RoomKeysUploadServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py new file mode 100644 index 000000000000..9f6d47e1b63b --- /dev/null +++ b/synapse/storage/e2e_room_keys.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from twisted.internet import defer + +from synapse.util.caches.descriptors import cached + +from canonicaljson import encode_canonical_json +import ujson as json + +from ._base import SQLBaseStore + + +class EndToEndRoomKeyStore(SQLBaseStore): + + @defer.inlineCallbacks + def get_e2e_room_key(self, user_id, version, room_id, session_id): + + row = yield self._simple_select_one( + table="e2e_room_keys", + keyvalues={ + "user_id": user_id, + "version": version, + "room_id": room_id, + "session_id": session_id, + }, + retcols=( + "first_message_index", + "forwarded_count", + "is_verified", + "session_data", + ), + desc="get_e2e_room_key", + ) + + defer.returnValue(row); + + def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): + + def _set_e2e_room_key_txn(txn): + + self._simple_upsert( + txn, + table="e2e_room_keys", + keyvalues={ + "user_id": user_id, + "room_id": room_id, + "session_id": session_id, + } + values=[ + { + "version": version, + "first_message_index": room_key['first_message_index'], + "forwarded_count": room_key['forwarded_count'], + "is_verified": room_key['is_verified'], + "session_data": room_key['session_data'], + } + ], + lock=False, + ) + + return True + + return self.runInteraction( + "set_e2e_room_key", _set_e2e_room_key_txn + ) + + + def set_e2e_room_keys(self, user_id, version, room_keys): + + def _set_e2e_room_keys_txn(txn): + + self._simple_insert_many_txn( + txn, + table="e2e_room_keys", + values=[ + { + "user_id": user_id, + "room_id": room_id, + "session_id": session_id, + "version": version, + "first_message_index": room_keys['rooms'][room_id]['sessions'][session_id]['first_message_index'], + "forwarded_count": room_keys['rooms'][room_id]['sessions'][session_id]['forwarded_count'], + "is_verified": room_keys['rooms'][room_id]['sessions'][session_id]['is_verified'], + "session_data": room_keys['rooms'][room_id]['sessions'][session_id]['session_data'], + } + for session_id in room_keys['rooms'][room_id]['sessions'] + for room_id in room_keys['rooms'] + ] + ) + + return True + + return self.runInteraction( + "set_e2e_room_keys", _set_e2e_room_keys_txn + ) + + @defer.inlineCallbacks + def get_e2e_room_keys(self, user_id, version, room_id, session_id): + + keyvalues={ + "user_id": user_id, + "version": version, + } + if room_id: keyvalues['room_id'] = room_id + if session_id: keyvalues['session_id'] = session_id + + rows = yield self._simple_select_list( + table="e2e_room_keys", + keyvalues=keyvalues, + retcols=( + "first_message_index", + "forwarded_count", + "is_verified", + "session_data", + ), + desc="get_e2e_room_keys", + ) + + sessions = {} + sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows; + defer.returnValue(sessions); diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql new file mode 100644 index 000000000000..51b826e8b3df --- /dev/null +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -0,0 +1,40 @@ +/* Copyright 2017 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- users' optionally backed up encrypted e2e sessions +CREATE TABLE e2e_room_keys ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + session_id TEXT NOT NULL, + version INT NOT NULL, + first_message_index INT, + forwarded_count INT, + is_verified BOOLEAN, + session_data TEXT NOT NULL +); + +CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id); +CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id); +CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id); + +-- the versioning metadata about versions of users' encrypted e2e session backups +CREATE TABLE e2e_room_key_versions ( + user_id TEXT NOT NULL, + version INT NOT NULL, + algorithm TEXT NOT NULL, + dummy_session_data TEXT NOT NULL +); + +CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id); From 0bc4627a731d0edd437905a5b07e85421c7553d8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 17:54:48 +0000 Subject: [PATCH 02/45] interim WIP checkin; doesn't build yet --- synapse/handlers/e2e_room_keys.py | 46 +++++++++++++++-------- synapse/rest/client/v2_alpha/room_keys.py | 37 ++++++++++++++++-- synapse/storage/e2e_room_keys.py | 20 ++++++++++ 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 78c838a82950..15e3beb5ed62 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -20,8 +20,7 @@ from twisted.internet import defer from synapse.api.errors import SynapseError, CodeMessageException -from synapse.types import get_domain_from_id -from synapse.util.logcontext import preserve_fn, make_deferred_yieldable +from synapse.util.async import Linearizer from synapse.util.retryutils import NotRetryingDestination logger = logging.getLogger(__name__) @@ -30,31 +29,48 @@ class E2eRoomKeysHandler(object): def __init__(self, hs): self.store = hs.get_datastore() + self._upload_linearizer = async.Linearizer("upload_room_keys_lock") @defer.inlineCallbacks def get_room_keys(self, user_id, version, room_id, session_id): results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id) defer.returnValue(results) + @defer.inlineCallbacks + def delete_room_keys(self, user_id, version, room_id, session_id): + yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) + @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): # TODO: Validate the JSON to make sure it has the right keys. - # go through the room_keys - for room_id in room_keys['rooms']: - for session_id in room_keys['rooms'][room_id]['sessions']: - session = room_keys['rooms'][room_id]['sessions'][session_id] - - # get a lock + # XXX: perhaps we should use a finer grained lock here? + with (yield self._upload_linearizer.queue(user_id): - # get the room_key for this particular row - yield self.store.get_e2e_room_key() + # go through the room_keys + for room_id in room_keys['rooms']: + for session_id in room_keys['rooms'][room_id]['sessions']: + room_key = room_keys['rooms'][room_id]['sessions'][session_id] - # check whether we merge or not - if() + # get the room_key for this particular row + current_room_key = yield self.store.get_e2e_room_key( + user_id, version, room_id, session_id + ) - # if so, we set it - yield self.store.set_e2e_room_key() + # check whether we merge or not. spelling it out with if/elifs rather than + # lots of booleans for legibility. + replace = False + if current_room_key: + if room_key['is_verified'] and not current_room_key['is_verified']: + replace = True + elif room_key['first_message_index'] < current_room_key['first_message_index']: + replace = True + elif room_key['forwarded_count'] < room_key['forwarded_count']: + replace = True - # release the lock + # if so, we set the new room_key + if replace: + yield self.store.set_e2e_room_key( + user_id, version, room_id, session_id, room_key + ) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 9b93001919d0..7291018a480a 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) -class RoomKeysUploadServlet(RestServlet): +class RoomKeysServlet(RestServlet): PATTERNS = client_v2_patterns("/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$") def __init__(self, hs): @@ -41,16 +41,45 @@ def __init__(self, hs): self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() @defer.inlineCallbacks - def on_POST(self, request, room_id, session_id): - requester = yield self.auth.get_user_by_req(request, allow_guest=True) + def on_PUT(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) + version = request.args.get("version", None) + + if session_id: + body = { "sessions": { session_id : body } } + + if room_id: + body = { "rooms": { room_id : body } } result = yield self.e2e_room_keys_handler.upload_room_keys( user_id, version, body ) defer.returnValue((200, result)) + @defer.inlineCallbacks + def on_GET(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + version = request.args.get("version", None) + + room_keys = yield self.e2e_room_keys_handler.get_room_keys( + user_id, version, room_id, session_id + ) + defer.returnValue((200, room_keys)) + + @defer.inlineCallbacks + def on_DELETE(self, request, room_id, session_id): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + version = request.args.get("version", None) + + yield self.e2e_room_keys_handler.delete_room_keys( + user_id, version, room_id, session_id + ) + defer.returnValue((200, {})) + def register_servlets(hs, http_server): - RoomKeysUploadServlet(hs).register(http_server) + RoomKeysServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 9f6d47e1b63b..903dc083f84a 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + from twisted.internet import defer from synapse.util.caches.descriptors import cached @@ -77,6 +78,9 @@ def _set_e2e_room_key_txn(txn): ) + # XXX: this isn't currently used and isn't tested anywhere + # it could be used in future for bulk-uploading new versions of room_keys + # for a user or something though. def set_e2e_room_keys(self, user_id, version, room_keys): def _set_e2e_room_keys_txn(txn): @@ -131,3 +135,19 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id): sessions = {} sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows; defer.returnValue(sessions); + + @defer.inlineCallbacks + def delete_e2e_room_keys(self, user_id, version, room_id, session_id): + + keyvalues={ + "user_id": user_id, + "version": version, + } + if room_id: keyvalues['room_id'] = room_id + if session_id: keyvalues['session_id'] = session_id + + yield self._simple_delete( + table="e2e_room_keys", + keyvalues=keyvalues, + desc="delete_e2e_room_keys", + ) From 6b8c07abc293bd222051e769550753bc0fd6f667 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 21:44:25 +0000 Subject: [PATCH 03/45] make it work and fix pep8 --- synapse/handlers/e2e_room_keys.py | 69 +++++++----- synapse/rest/__init__.py | 2 + synapse/rest/client/v2_alpha/room_keys.py | 33 +++--- synapse/server.py | 5 + synapse/storage/__init__.py | 2 + synapse/storage/e2e_room_keys.py | 103 +++++++++++------- .../storage/schema/delta/46/e2e_room_keys.sql | 2 +- 7 files changed, 134 insertions(+), 82 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 15e3beb5ed62..93f4ad519481 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -13,15 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ujson as json import logging -from canonicaljson import encode_canonical_json from twisted.internet import defer -from synapse.api.errors import SynapseError, CodeMessageException +from synapse.api.errors import StoreError from synapse.util.async import Linearizer -from synapse.util.retryutils import NotRetryingDestination logger = logging.getLogger(__name__) @@ -29,11 +26,13 @@ class E2eRoomKeysHandler(object): def __init__(self, hs): self.store = hs.get_datastore() - self._upload_linearizer = async.Linearizer("upload_room_keys_lock") + self._upload_linearizer = Linearizer("upload_room_keys_lock") @defer.inlineCallbacks def get_room_keys(self, user_id, version, room_id, session_id): - results = yield self.store.get_e2e_room_keys(user_id, version, room_id, session_id) + results = yield self.store.get_e2e_room_keys( + user_id, version, room_id, session_id + ) defer.returnValue(results) @defer.inlineCallbacks @@ -46,31 +45,49 @@ def upload_room_keys(self, user_id, version, room_keys): # TODO: Validate the JSON to make sure it has the right keys. # XXX: perhaps we should use a finer grained lock here? - with (yield self._upload_linearizer.queue(user_id): + with (yield self._upload_linearizer.queue(user_id)): # go through the room_keys for room_id in room_keys['rooms']: for session_id in room_keys['rooms'][room_id]['sessions']: room_key = room_keys['rooms'][room_id]['sessions'][session_id] - # get the room_key for this particular row - current_room_key = yield self.store.get_e2e_room_key( - user_id, version, room_id, session_id + yield self._upload_room_key( + user_id, version, room_id, session_id, room_key ) - # check whether we merge or not. spelling it out with if/elifs rather than - # lots of booleans for legibility. - replace = False - if current_room_key: - if room_key['is_verified'] and not current_room_key['is_verified']: - replace = True - elif room_key['first_message_index'] < current_room_key['first_message_index']: - replace = True - elif room_key['forwarded_count'] < room_key['forwarded_count']: - replace = True - - # if so, we set the new room_key - if replace: - yield self.store.set_e2e_room_key( - user_id, version, room_id, session_id, room_key - ) + @defer.inlineCallbacks + def _upload_room_key(self, user_id, version, room_id, session_id, room_key): + # get the room_key for this particular row + current_room_key = None + try: + current_room_key = yield self.store.get_e2e_room_key( + user_id, version, room_id, session_id + ) + except StoreError as e: + if e.code == 404: + pass + else: + raise + + # check whether we merge or not. spelling it out with if/elifs rather + # than lots of booleans for legibility. + upsert = True + if current_room_key: + if room_key['is_verified'] and not current_room_key['is_verified']: + pass + elif ( + room_key['first_message_index'] < + current_room_key['first_message_index'] + ): + pass + elif room_key['forwarded_count'] < room_key['forwarded_count']: + pass + else: + upsert = False + + # if so, we set the new room_key + if upsert: + yield self.store.set_e2e_room_key( + user_id, version, room_id, session_id, room_key + ) diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 3418f06fd6e5..4856822a5da5 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -46,6 +46,7 @@ receipts, register, report_event, + room_keys, sendtodevice, sync, tags, @@ -102,6 +103,7 @@ def register_servlets(client_resource, hs): auth.register_servlets(hs, client_resource) receipts.register_servlets(hs, client_resource) read_marker.register_servlets(hs, client_resource) + room_keys.register_servlets(hs, client_resource) keys.register_servlets(hs, client_resource) tokenrefresh.register_servlets(hs, client_resource) tags.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 7291018a480a..010aed98f924 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,26 +17,25 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request, parse_integer + RestServlet, parse_json_object_from_request ) -from synapse.http.servlet import parse_string -from synapse.types import StreamToken from ._base import client_v2_patterns logger = logging.getLogger(__name__) class RoomKeysServlet(RestServlet): - PATTERNS = client_v2_patterns("/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$") + PATTERNS = client_v2_patterns( + "/room_keys/keys(/(?P[^/]+))?(/(?P[^/]+))?$" + ) def __init__(self, hs): """ Args: hs (synapse.server.HomeServer): server """ - super(RoomKeysUploadServlet, self).__init__() + super(RoomKeysServlet, self).__init__() self.auth = hs.get_auth() self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() @@ -45,24 +44,32 @@ def on_PUT(self, request, room_id, session_id): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) - version = request.args.get("version", None) + version = request.args.get("version")[0] if session_id: - body = { "sessions": { session_id : body } } + body = { + "sessions": { + session_id: body + } + } if room_id: - body = { "rooms": { room_id : body } } + body = { + "rooms": { + room_id: body + } + } - result = yield self.e2e_room_keys_handler.upload_room_keys( + yield self.e2e_room_keys_handler.upload_room_keys( user_id, version, body ) - defer.returnValue((200, result)) + defer.returnValue((200, {})) @defer.inlineCallbacks def on_GET(self, request, room_id, session_id): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version", None) + version = request.args.get("version")[0] room_keys = yield self.e2e_room_keys_handler.get_room_keys( user_id, version, room_id, session_id @@ -73,7 +80,7 @@ def on_GET(self, request, room_id, session_id): def on_DELETE(self, request, room_id, session_id): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version", None) + version = request.args.get("version")[0] yield self.e2e_room_keys_handler.delete_room_keys( user_id, version, room_id, session_id diff --git a/synapse/server.py b/synapse/server.py index 140be9ebe863..706cb1361f59 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -49,6 +49,7 @@ from synapse.handlers.device import DeviceHandler from synapse.handlers.devicemessage import DeviceMessageHandler from synapse.handlers.e2e_keys import E2eKeysHandler +from synapse.handlers.e2e_room_keys import E2eRoomKeysHandler from synapse.handlers.events import EventHandler, EventStreamHandler from synapse.handlers.groups_local import GroupsLocalHandler from synapse.handlers.initial_sync import InitialSyncHandler @@ -127,6 +128,7 @@ def build_DEPENDENCY(self) 'auth_handler', 'device_handler', 'e2e_keys_handler', + 'e2e_room_keys_handler', 'event_handler', 'event_stream_handler', 'initial_sync_handler', @@ -288,6 +290,9 @@ def build_device_message_handler(self): def build_e2e_keys_handler(self): return E2eKeysHandler(self) + def build_e2e_room_keys_handler(self): + return E2eRoomKeysHandler(self) + def build_application_service_api(self): return ApplicationServiceApi(self) diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py index 134e4a80f1f0..69cb28268ac0 100644 --- a/synapse/storage/__init__.py +++ b/synapse/storage/__init__.py @@ -30,6 +30,7 @@ from .client_ips import ClientIpStore from .deviceinbox import DeviceInboxStore from .directory import DirectoryStore +from .e2e_room_keys import EndToEndRoomKeyStore from .end_to_end_keys import EndToEndKeyStore from .engines import PostgresEngine from .event_federation import EventFederationStore @@ -76,6 +77,7 @@ class DataStore(RoomMemberStore, RoomStore, ApplicationServiceTransactionStore, ReceiptsStore, EndToEndKeyStore, + EndToEndRoomKeyStore, SearchStore, TagsStore, AccountDataStore, diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 903dc083f84a..5982710bd574 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -15,11 +15,6 @@ from twisted.internet import defer -from synapse.util.caches.descriptors import cached - -from canonicaljson import encode_canonical_json -import ujson as json - from ._base import SQLBaseStore @@ -45,29 +40,27 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id): desc="get_e2e_room_key", ) - defer.returnValue(row); + defer.returnValue(row) def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): def _set_e2e_room_key_txn(txn): - self._simple_upsert( + self._simple_upsert_txn( txn, table="e2e_room_keys", keyvalues={ "user_id": user_id, "room_id": room_id, - "session_id": session_id, - } - values=[ - { - "version": version, - "first_message_index": room_key['first_message_index'], - "forwarded_count": room_key['forwarded_count'], - "is_verified": room_key['is_verified'], - "session_data": room_key['session_data'], - } - ], + "session_id": session_id, + }, + values={ + "version": version, + "first_message_index": room_key['first_message_index'], + "forwarded_count": room_key['forwarded_count'], + "is_verified": room_key['is_verified'], + "session_data": room_key['session_data'], + }, lock=False, ) @@ -77,7 +70,6 @@ def _set_e2e_room_key_txn(txn): "set_e2e_room_key", _set_e2e_room_key_txn ) - # XXX: this isn't currently used and isn't tested anywhere # it could be used in future for bulk-uploading new versions of room_keys # for a user or something though. @@ -85,23 +77,27 @@ def set_e2e_room_keys(self, user_id, version, room_keys): def _set_e2e_room_keys_txn(txn): + values = [] + for room_id in room_keys['rooms']: + for session_id in room_keys['rooms'][room_id]['sessions']: + session = room_keys['rooms'][room_id]['sessions'][session_id] + values.append( + { + "user_id": user_id, + "room_id": room_id, + "session_id": session_id, + "version": version, + "first_message_index": session['first_message_index'], + "forwarded_count": session['forwarded_count'], + "is_verified": session['is_verified'], + "session_data": session['session_data'], + } + ) + self._simple_insert_many_txn( txn, table="e2e_room_keys", - values=[ - { - "user_id": user_id, - "room_id": room_id, - "session_id": session_id, - "version": version, - "first_message_index": room_keys['rooms'][room_id]['sessions'][session_id]['first_message_index'], - "forwarded_count": room_keys['rooms'][room_id]['sessions'][session_id]['forwarded_count'], - "is_verified": room_keys['rooms'][room_id]['sessions'][session_id]['is_verified'], - "session_data": room_keys['rooms'][room_id]['sessions'][session_id]['session_data'], - } - for session_id in room_keys['rooms'][room_id]['sessions'] - for room_id in room_keys['rooms'] - ] + values=values ) return True @@ -113,17 +109,22 @@ def _set_e2e_room_keys_txn(txn): @defer.inlineCallbacks def get_e2e_room_keys(self, user_id, version, room_id, session_id): - keyvalues={ + keyvalues = { "user_id": user_id, "version": version, } - if room_id: keyvalues['room_id'] = room_id - if session_id: keyvalues['session_id'] = session_id + if room_id: + keyvalues['room_id'] = room_id + if session_id: + keyvalues['session_id'] = session_id rows = yield self._simple_select_list( table="e2e_room_keys", keyvalues=keyvalues, retcols=( + "user_id", + "room_id", + "session_id", "first_message_index", "forwarded_count", "is_verified", @@ -132,19 +133,37 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id): desc="get_e2e_room_keys", ) - sessions = {} - sessions['rooms'][roomId]['sessions'][session_id] = row for row in rows; - defer.returnValue(sessions); + # perlesque autovivification from https://stackoverflow.com/a/19829714/6764493 + class AutoVivification(dict): + def __getitem__(self, item): + try: + return dict.__getitem__(self, item) + except KeyError: + value = self[item] = type(self)() + return value + + sessions = AutoVivification() + for row in rows: + sessions['rooms'][row['room_id']]['sessions'][row['session_id']] = { + "first_message_index": row["first_message_index"], + "forwarded_count": row["forwarded_count"], + "is_verified": row["is_verified"], + "session_data": row["session_data"], + } + + defer.returnValue(sessions) @defer.inlineCallbacks def delete_e2e_room_keys(self, user_id, version, room_id, session_id): - keyvalues={ + keyvalues = { "user_id": user_id, "version": version, } - if room_id: keyvalues['room_id'] = room_id - if session_id: keyvalues['session_id'] = session_id + if room_id: + keyvalues['room_id'] = room_id + if session_id: + keyvalues['session_id'] = session_id yield self._simple_delete( table="e2e_room_keys", diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 51b826e8b3df..6b344c5ad78d 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -29,7 +29,7 @@ CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id); CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id); CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id); --- the versioning metadata about versions of users' encrypted e2e session backups +-- the metadata for each generation of encrypted e2e session backups CREATE TABLE e2e_room_key_versions ( user_id TEXT NOT NULL, version INT NOT NULL, From cf1e2000f623afea8f3afb58e4a7659288c45777 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 5 Dec 2017 23:06:43 +0000 Subject: [PATCH 04/45] document the API --- synapse/rest/client/v2_alpha/room_keys.py | 133 ++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 010aed98f924..be82eccb2bc8 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -41,6 +41,79 @@ def __init__(self, hs): @defer.inlineCallbacks def on_PUT(self, request, room_id, session_id): + """ + Uploads one or more encrypted E2E room keys for backup purposes. + room_id: the ID of the room the keys are for (optional) + session_id: the ID for the E2E room keys for the room (optional) + version: the version of the user's backup which this data is for. + the version must already have been created via the /change_secret API. + + Each session has: + * first_message_index: a numeric index indicating the oldest message + encrypted by this session. + * forwarded_count: how many times the uploading client claims this key + has been shared (forwarded) + * is_verified: whether the client that uploaded the keys claims they + were sent by a device which they've verified + * session_data: base64-encrypted data describing the session. + + Returns 200 OK on success with body {} + + The API is designed to be otherwise agnostic to the room_key encryption + algorithm being used. Sessions are merged with existing ones in the + backup using the heuristics: + * is_verified sessions always win over unverified sessions + * older first_message_index always win over newer sessions + * lower forwarded_count always wins over higher forwarded_count + + We trust the clients not to lie and corrupt their own backups. + + POST /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1 + Content-Type: application/json + + { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + + Or... + + POST /room_keys/keys/!abc:matrix.org?version=1 HTTP/1.1 + Content-Type: application/json + + { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + + Or... + + POST /room_keys/keys?version=1 HTTP/1.1 + Content-Type: application/json + + { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } + } + """ requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) @@ -67,6 +140,57 @@ def on_PUT(self, request, room_id, session_id): @defer.inlineCallbacks def on_GET(self, request, room_id, session_id): + """ + Retrieves one or more encrypted E2E room keys for backup purposes. + Symmetric with the PUT version of the API. + + room_id: the ID of the room to retrieve the keys for (optional) + session_id: the ID for the E2E room keys to retrieve the keys for (optional) + version: the version of the user's backup which this data is for. + the version must already have been created via the /change_secret API. + + Returns as follows: + + GET /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1 + { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + + Or... + + GET /room_keys/keys/!abc:matrix.org?version=1 HTTP/1.1 + { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + + Or... + + GET /room_keys/keys?version=1 HTTP/1.1 + { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } + } + """ requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() version = request.args.get("version")[0] @@ -78,6 +202,15 @@ def on_GET(self, request, room_id, session_id): @defer.inlineCallbacks def on_DELETE(self, request, room_id, session_id): + """ + Deletes one or more encrypted E2E room keys for a user for backup purposes. + + room_id: the ID of the room whose keys to delete (optional) + session_id: the ID for the E2E session to delete (optional) + version: the version of the user's backup which this data is for. + the version must already have been created via the /change_secret API. + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() version = request.args.get("version")[0] From 8ae64b270f7742abfe4cf0b8d140c81993464ea4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 6 Dec 2017 01:02:57 +0000 Subject: [PATCH 05/45] implement /room_keys/version too (untested) --- synapse/api/errors.py | 25 +++++++++ synapse/handlers/e2e_room_keys.py | 47 ++++++++++++++-- synapse/rest/client/v2_alpha/room_keys.py | 47 ++++++++++++++++ synapse/storage/e2e_room_keys.py | 56 +++++++++++++++++++ .../storage/schema/delta/46/e2e_room_keys.sql | 2 +- 5 files changed, 171 insertions(+), 6 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index b41d595059a3..8c97e91ba194 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -56,6 +56,7 @@ class Codes(object): CONSENT_NOT_GIVEN = "M_CONSENT_NOT_GIVEN" CANNOT_LEAVE_SERVER_NOTICE_ROOM = "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" MAU_LIMIT_EXCEEDED = "M_MAU_LIMIT_EXCEEDED" + WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION" class CodeMessageException(RuntimeError): @@ -285,6 +286,30 @@ def error_dict(self): ) +class RoomKeysVersionError(SynapseError): + """A client has tried to upload to a non-current version of the room_keys store + """ + def __init__(self, code=403, msg="Wrong room_keys version", current_version=None, + errcode=Codes.WRONG_ROOM_KEYS_VERSION): + super(RoomKeysVersionError, self).__init__(code, msg, errcode) + self.current_version = current_version + + def error_dict(self): + return cs_error( + self.msg, + self.errcode, + current_version=self.current_version, + ) + + +def cs_exception(exception): + if isinstance(exception, CodeMessageException): + return exception.error_dict() + else: + logger.error("Unknown exception type: %s", type(exception)) + return {} + + def cs_error(msg, code=Codes.UNKNOWN, **kwargs): """ Utility method for constructing an error response for client-server interactions. diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 93f4ad519481..4333ca610cbe 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -17,7 +17,7 @@ from twisted.internet import defer -from synapse.api.errors import StoreError +from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError from synapse.util.async import Linearizer logger = logging.getLogger(__name__) @@ -30,10 +30,13 @@ def __init__(self, hs): @defer.inlineCallbacks def get_room_keys(self, user_id, version, room_id, session_id): - results = yield self.store.get_e2e_room_keys( - user_id, version, room_id, session_id - ) - defer.returnValue(results) + # we deliberately take the lock to get keys so that changing the version + # works atomically + with (yield self._upload_linearizer.queue(user_id)): + results = yield self.store.get_e2e_room_keys( + user_id, version, room_id, session_id + ) + defer.returnValue(results) @defer.inlineCallbacks def delete_room_keys(self, user_id, version, room_id, session_id): @@ -44,6 +47,16 @@ def upload_room_keys(self, user_id, version, room_keys): # TODO: Validate the JSON to make sure it has the right keys. + # Check that the version we're trying to upload is the current version + try: + version_info = yield self.get_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Version '%d' not found" % (version,)) + + if version_info.version != version: + raise RoomKeysVersionError(current_version=version_info.version) + # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): @@ -91,3 +104,27 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): yield self.store.set_e2e_room_key( user_id, version, room_id, session_id, room_key ) + + @defer.inlineCallbacks + def create_version(self, user_id, version, version_info): + + # TODO: Validate the JSON to make sure it has the right keys. + + # lock everyone out until we've switched version + with (yield self._upload_linearizer.queue(user_id)): + yield self.store.create_version( + user_id, version, version_info + ) + + @defer.inlineCallbacks + def get_version_info(self, user_id, version): + with (yield self._upload_linearizer.queue(user_id)): + results = yield self.store.get_e2e_room_key_version( + user_id, version + ) + defer.returnValue(results) + + @defer.inlineCallbacks + def delete_version(self, user_id, version): + with (yield self._upload_linearizer.queue(user_id)): + yield self.store.delete_e2e_room_key_version(user_id, version) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index be82eccb2bc8..4d76e1d824dc 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -221,5 +221,52 @@ def on_DELETE(self, request, room_id, session_id): defer.returnValue((200, {})) +class RoomKeysVersionServlet(RestServlet): + PATTERNS = client_v2_patterns( + "/room_keys/version(/(?P[^/]+))?$" + ) + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(RoomKeysVersionServlet, self).__init__() + self.auth = hs.get_auth() + self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() + + @defer.inlineCallbacks + def on_POST(self, request, version): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + info = parse_json_object_from_request(request) + + new_version = yield self.e2e_room_keys_handler.create_version( + user_id, version, info + ) + defer.returnValue((200, {"version": new_version})) + + @defer.inlineCallbacks + def on_GET(self, request, version): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + + info = yield self.e2e_room_keys_handler.get_version_info( + user_id, version + ) + defer.returnValue((200, info)) + + @defer.inlineCallbacks + def on_DELETE(self, request, version): + requester = yield self.auth.get_user_by_req(request, allow_guest=False) + user_id = requester.user.to_string() + + yield self.e2e_room_keys_handler.delete_version( + user_id, version + ) + defer.returnValue((200, {})) + + def register_servlets(hs, http_server): RoomKeysServlet(hs).register(http_server) + RoomKeysVersionServlet(hs).register(http_server) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 5982710bd574..994878acf684 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -170,3 +170,59 @@ def delete_e2e_room_keys(self, user_id, version, room_id, session_id): keyvalues=keyvalues, desc="delete_e2e_room_keys", ) + + @defer.inlineCallbacks + def get_e2e_room_key_version(self, user_id, version): + + row = yield self._simple_select_one( + table="e2e_room_key_versions", + keyvalues={ + "user_id": user_id, + "version": version, + }, + retcols=( + "user_id", + "version", + "algorithm", + "auth_data", + ), + desc="get_e2e_room_key_version_info", + ) + + defer.returnValue(row) + + def create_e2e_room_key_version(self, user_id, version, info): + + def _create_e2e_room_key_version_txn(txn): + + self._simple_insert_txn( + txn, + table="e2e_room_key_versions", + values={ + "user_id": user_id, + "version": version, + "algorithm": info["algorithm"], + "auth_data": info["auth_data"], + }, + lock=False, + ) + + return True + + return self.runInteraction( + "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn + ) + + @defer.inlineCallbacks + def delete_e2e_room_key_version(self, user_id, version): + + keyvalues = { + "user_id": user_id, + "version": version, + } + + yield self._simple_delete( + table="e2e_room_key_versions", + keyvalues=keyvalues, + desc="delete_e2e_room_key_version", + ) diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 6b344c5ad78d..463f828c6670 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -34,7 +34,7 @@ CREATE TABLE e2e_room_key_versions ( user_id TEXT NOT NULL, version INT NOT NULL, algorithm TEXT NOT NULL, - dummy_session_data TEXT NOT NULL + auth_data TEXT NOT NULL ); CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id); From 69e51c7ba48a84b48ab64c8c290a232d14193a18 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 6 Dec 2017 10:02:49 +0100 Subject: [PATCH 06/45] make /room_keys/version work --- synapse/handlers/e2e_room_keys.py | 18 +++++++++------ synapse/rest/client/v2_alpha/room_keys.py | 9 +++++++- synapse/storage/e2e_room_keys.py | 22 ++++++++++++++----- .../storage/schema/delta/46/e2e_room_keys.sql | 4 ++-- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 4333ca610cbe..bd58be655853 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -48,13 +48,16 @@ def upload_room_keys(self, user_id, version, room_keys): # TODO: Validate the JSON to make sure it has the right keys. # Check that the version we're trying to upload is the current version + try: version_info = yield self.get_version_info(user_id, version) except StoreError as e: if e.code == 404: - raise SynapseError(404, "Version '%d' not found" % (version,)) + raise SynapseError(404, "Version '%s' not found" % (version,)) + else: + raise e - if version_info.version != version: + if version_info['version'] != version: raise RoomKeysVersionError(current_version=version_info.version) # XXX: perhaps we should use a finer grained lock here? @@ -81,7 +84,7 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): if e.code == 404: pass else: - raise + raise e # check whether we merge or not. spelling it out with if/elifs rather # than lots of booleans for legibility. @@ -106,20 +109,21 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): ) @defer.inlineCallbacks - def create_version(self, user_id, version, version_info): + def create_version(self, user_id, version_info): # TODO: Validate the JSON to make sure it has the right keys. # lock everyone out until we've switched version with (yield self._upload_linearizer.queue(user_id)): - yield self.store.create_version( - user_id, version, version_info + new_version = yield self.store.create_e2e_room_key_version( + user_id, version_info ) + defer.returnValue(new_version) @defer.inlineCallbacks def get_version_info(self, user_id, version): with (yield self._upload_linearizer.queue(user_id)): - results = yield self.store.get_e2e_room_key_version( + results = yield self.store.get_e2e_room_key_version_info( user_id, version ) defer.returnValue(results) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 4d76e1d824dc..128b732fb15c 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,6 +17,7 @@ from twisted.internet import defer +from synapse.api.errors import SynapseError from synapse.http.servlet import ( RestServlet, parse_json_object_from_request ) @@ -237,15 +238,21 @@ def __init__(self, hs): @defer.inlineCallbacks def on_POST(self, request, version): + if version: + raise SynapseError(405, "Cannot POST to a specific version") + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() info = parse_json_object_from_request(request) new_version = yield self.e2e_room_keys_handler.create_version( - user_id, version, info + user_id, info ) defer.returnValue((200, {"version": new_version})) + # we deliberately don't have a PUT /version, as these things really should + # be immutable to avoid people footgunning + @defer.inlineCallbacks def on_GET(self, request, version): requester = yield self.auth.get_user_by_req(request, allow_guest=False) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 994878acf684..8efca11a8cc5 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -172,7 +172,7 @@ def delete_e2e_room_keys(self, user_id, version, room_id, session_id): ) @defer.inlineCallbacks - def get_e2e_room_key_version(self, user_id, version): + def get_e2e_room_key_version_info(self, user_id, version): row = yield self._simple_select_one( table="e2e_room_key_versions", @@ -191,23 +191,35 @@ def get_e2e_room_key_version(self, user_id, version): defer.returnValue(row) - def create_e2e_room_key_version(self, user_id, version, info): + def create_e2e_room_key_version(self, user_id, info): + """Atomically creates a new version of this user's e2e_room_keys store + with the given version info. + """ def _create_e2e_room_key_version_txn(txn): + txn.execute( + "SELECT MAX(version) FROM e2e_room_key_versions WHERE user_id=?", + (user_id,) + ) + current_version = txn.fetchone()[0] + if current_version is None: + current_version = 0 + + new_version = current_version + 1 + self._simple_insert_txn( txn, table="e2e_room_key_versions", values={ "user_id": user_id, - "version": version, + "version": new_version, "algorithm": info["algorithm"], "auth_data": info["auth_data"], }, - lock=False, ) - return True + return new_version return self.runInteraction( "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 463f828c6670..0d2a85fbe67e 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -18,7 +18,7 @@ CREATE TABLE e2e_room_keys ( user_id TEXT NOT NULL, room_id TEXT NOT NULL, session_id TEXT NOT NULL, - version INT NOT NULL, + version TEXT NOT NULL, first_message_index INT, forwarded_count INT, is_verified BOOLEAN, @@ -32,7 +32,7 @@ CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id); -- the metadata for each generation of encrypted e2e session backups CREATE TABLE e2e_room_key_versions ( user_id TEXT NOT NULL, - version INT NOT NULL, + version TEXT NOT NULL, algorithm TEXT NOT NULL, auth_data TEXT NOT NULL ); From 0abb205b47158a4160ddceb317c0245d640b6e3f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 18 Dec 2017 01:52:46 +0000 Subject: [PATCH 07/45] blindly incorporate PR review - needs testing & fixing --- synapse/api/errors.py | 11 ++- synapse/handlers/e2e_room_keys.py | 88 ++++++++++++------- synapse/rest/client/v2_alpha/room_keys.py | 2 + synapse/storage/e2e_room_keys.py | 69 ++++++--------- .../storage/schema/delta/46/e2e_room_keys.sql | 8 +- 5 files changed, 99 insertions(+), 79 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 8c97e91ba194..d37bcb4082ba 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -289,9 +289,14 @@ def error_dict(self): class RoomKeysVersionError(SynapseError): """A client has tried to upload to a non-current version of the room_keys store """ - def __init__(self, code=403, msg="Wrong room_keys version", current_version=None, - errcode=Codes.WRONG_ROOM_KEYS_VERSION): - super(RoomKeysVersionError, self).__init__(code, msg, errcode) + def __init__(self, current_version): + """ + Args: + current_version (str): the current version of the store they should have used + """ + super(RoomKeysVersionError, self).__init__( + 403, "Wrong room_keys version", Codes.WRONG_ROOM_KEYS_VERSION + ) self.current_version = current_version def error_dict(self): diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index bd58be655853..dda31fdd24dc 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -24,8 +24,21 @@ class E2eRoomKeysHandler(object): + """ + Implements an optional realtime backup mechanism for encrypted E2E megolm room keys. + This gives a way for users to store and recover their megolm keys if they lose all + their clients. It should also extend easily to future room key mechanisms. + The actual payload of the encrypted keys is completely opaque to the handler. + """ + def __init__(self, hs): self.store = hs.get_datastore() + + # Used to lock whenever a client is uploading key data. This prevents collisions + # between clients trying to upload the details of a new session, given all + # clients belonging to a user will receive and try to upload a new session at + # roughly the same time. Also used to lock out uploads when the key is being + # changed. self._upload_linearizer = Linearizer("upload_room_keys_lock") @defer.inlineCallbacks @@ -40,33 +53,34 @@ def get_room_keys(self, user_id, version, room_id, session_id): @defer.inlineCallbacks def delete_room_keys(self, user_id, version, room_id, session_id): - yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) + # lock for consistency with uploading + with (yield self._upload_linearizer.queue(user_id)): + yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): # TODO: Validate the JSON to make sure it has the right keys. - # Check that the version we're trying to upload is the current version - - try: - version_info = yield self.get_version_info(user_id, version) - except StoreError as e: - if e.code == 404: - raise SynapseError(404, "Version '%s' not found" % (version,)) - else: - raise e - - if version_info['version'] != version: - raise RoomKeysVersionError(current_version=version_info.version) - # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): - - # go through the room_keys - for room_id in room_keys['rooms']: - for session_id in room_keys['rooms'][room_id]['sessions']: - room_key = room_keys['rooms'][room_id]['sessions'][session_id] + # Check that the version we're trying to upload is the current version + try: + version_info = yield self.get_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Version '%s' not found" % (version,)) + else: + raise e + + if version_info['version'] != version: + raise RoomKeysVersionError(current_version=version_info.version) + + # go through the room_keys. + # XXX: this should/could be done concurrently, given we're in a lock. + for room_id, room in room_keys['rooms'].iteritems(): + for session_id, session in room['sessions'].iteritems(): + room_key = session[session_id] yield self._upload_room_key( user_id, version, room_id, session_id, room_key @@ -86,10 +100,29 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): else: raise e - # check whether we merge or not. spelling it out with if/elifs rather - # than lots of booleans for legibility. - upsert = True + if _should_replace_room_key(current_room_key, room_key): + yield self.store.set_e2e_room_key( + user_id, version, room_id, session_id, room_key + ) + + def _should_replace_room_key(current_room_key, room_key): + """ + Determine whether to replace the current_room_key in our backup for this + session (if any) with a new room_key that has been uploaded. + + Args: + current_room_key (dict): Optional, the current room_key dict if any + room_key (dict): The new room_key dict which may or may not be fit to + replace the current_room_key + + Returns: + True if current_room_key should be replaced by room_key in the backup + """ + if current_room_key: + # spelt out with if/elifs rather than nested boolean expressions + # purely for legibility. + if room_key['is_verified'] and not current_room_key['is_verified']: pass elif ( @@ -97,16 +130,11 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): current_room_key['first_message_index'] ): pass - elif room_key['forwarded_count'] < room_key['forwarded_count']: + elif room_key['forwarded_count'] < current_room_key['forwarded_count']: pass else: - upsert = False - - # if so, we set the new room_key - if upsert: - yield self.store.set_e2e_room_key( - user_id, version, room_id, session_id, room_key - ) + return False + return True @defer.inlineCallbacks def create_version(self, user_id, version_info): diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 128b732fb15c..70b7b4573f57 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -68,6 +68,8 @@ def on_PUT(self, request, room_id, session_id): * lower forwarded_count always wins over higher forwarded_count We trust the clients not to lie and corrupt their own backups. + It also means that if your access_token is stolen, the attacker could + delete your backup. POST /room_keys/keys/!abc:matrix.org/c0ff33?version=1 HTTP/1.1 Content-Type: application/json diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 8efca11a8cc5..c11417c41539 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -44,30 +44,21 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id): def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): - def _set_e2e_room_key_txn(txn): - - self._simple_upsert_txn( - txn, - table="e2e_room_keys", - keyvalues={ - "user_id": user_id, - "room_id": room_id, - "session_id": session_id, - }, - values={ - "version": version, - "first_message_index": room_key['first_message_index'], - "forwarded_count": room_key['forwarded_count'], - "is_verified": room_key['is_verified'], - "session_data": room_key['session_data'], - }, - lock=False, - ) - - return True - - return self.runInteraction( - "set_e2e_room_key", _set_e2e_room_key_txn + yield self._simple_upsert( + table="e2e_room_keys", + keyvalues={ + "user_id": user_id, + "room_id": room_id, + "session_id": session_id, + }, + values={ + "version": version, + "first_message_index": room_key['first_message_index'], + "forwarded_count": room_key['forwarded_count'], + "is_verified": room_key['is_verified'], + "session_data": room_key['session_data'], + }, + lock=False, ) # XXX: this isn't currently used and isn't tested anywhere @@ -107,7 +98,9 @@ def _set_e2e_room_keys_txn(txn): ) @defer.inlineCallbacks - def get_e2e_room_keys(self, user_id, version, room_id, session_id): + def get_e2e_room_keys( + self, user_id, version, room_id=room_id, session_id=session_id + ): keyvalues = { "user_id": user_id, @@ -115,8 +108,8 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id): } if room_id: keyvalues['room_id'] = room_id - if session_id: - keyvalues['session_id'] = session_id + if session_id: + keyvalues['session_id'] = session_id rows = yield self._simple_select_list( table="e2e_room_keys", @@ -133,18 +126,10 @@ def get_e2e_room_keys(self, user_id, version, room_id, session_id): desc="get_e2e_room_keys", ) - # perlesque autovivification from https://stackoverflow.com/a/19829714/6764493 - class AutoVivification(dict): - def __getitem__(self, item): - try: - return dict.__getitem__(self, item) - except KeyError: - value = self[item] = type(self)() - return value - - sessions = AutoVivification() + sessions = {} for row in rows: - sessions['rooms'][row['room_id']]['sessions'][row['session_id']] = { + room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}}) + room_entry['sessions'][row['session_id']] = { "first_message_index": row["first_message_index"], "forwarded_count": row["forwarded_count"], "is_verified": row["is_verified"], @@ -154,7 +139,9 @@ def __getitem__(self, item): defer.returnValue(sessions) @defer.inlineCallbacks - def delete_e2e_room_keys(self, user_id, version, room_id, session_id): + def delete_e2e_room_keys( + self, user_id, version, room_id=room_id, session_id=session_id + ): keyvalues = { "user_id": user_id, @@ -162,8 +149,8 @@ def delete_e2e_room_keys(self, user_id, version, room_id, session_id): } if room_id: keyvalues['room_id'] = room_id - if session_id: - keyvalues['session_id'] = session_id + if session_id: + keyvalues['session_id'] = session_id yield self._simple_delete( table="e2e_room_keys", diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 0d2a85fbe67e..16499ac34c23 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -25,16 +25,14 @@ CREATE TABLE e2e_room_keys ( session_data TEXT NOT NULL ); -CREATE UNIQUE INDEX e2e_room_keys_user_idx ON e2e_room_keys(user_id); -CREATE UNIQUE INDEX e2e_room_keys_room_idx ON e2e_room_keys(room_id); -CREATE UNIQUE INDEX e2e_room_keys_session_idx ON e2e_room_keys(session_id); +CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id); -- the metadata for each generation of encrypted e2e session backups -CREATE TABLE e2e_room_key_versions ( +CREATE TABLE e2e_room_keys_versions ( user_id TEXT NOT NULL, version TEXT NOT NULL, algorithm TEXT NOT NULL, auth_data TEXT NOT NULL ); -CREATE UNIQUE INDEX e2e_room_key_user_idx ON e2e_room_keys(user_id); +CREATE UNIQUE INDEX e2e_room_keys_versions_user_idx ON e2e_room_keys_versions(user_id); From cac02537998718c05a561918269745161378dd6f Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 18 Dec 2017 01:58:53 +0000 Subject: [PATCH 08/45] rename room_key_version table correctly, and fix opt args --- synapse/handlers/e2e_room_keys.py | 6 +++--- synapse/storage/e2e_room_keys.py | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index dda31fdd24dc..87be081b1cfd 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -143,7 +143,7 @@ def create_version(self, user_id, version_info): # lock everyone out until we've switched version with (yield self._upload_linearizer.queue(user_id)): - new_version = yield self.store.create_e2e_room_key_version( + new_version = yield self.store.create_e2e_room_keys_version( user_id, version_info ) defer.returnValue(new_version) @@ -151,7 +151,7 @@ def create_version(self, user_id, version_info): @defer.inlineCallbacks def get_version_info(self, user_id, version): with (yield self._upload_linearizer.queue(user_id)): - results = yield self.store.get_e2e_room_key_version_info( + results = yield self.store.get_e2e_room_keys_version_info( user_id, version ) defer.returnValue(results) @@ -159,4 +159,4 @@ def get_version_info(self, user_id, version): @defer.inlineCallbacks def delete_version(self, user_id, version): with (yield self._upload_linearizer.queue(user_id)): - yield self.store.delete_e2e_room_key_version(user_id, version) + yield self.store.delete_e2e_room_keys_version(user_id, version) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index c11417c41539..7e1cb13e7427 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -99,7 +99,7 @@ def _set_e2e_room_keys_txn(txn): @defer.inlineCallbacks def get_e2e_room_keys( - self, user_id, version, room_id=room_id, session_id=session_id + self, user_id, version, room_id=None, session_id=None ): keyvalues = { @@ -140,7 +140,7 @@ def get_e2e_room_keys( @defer.inlineCallbacks def delete_e2e_room_keys( - self, user_id, version, room_id=room_id, session_id=session_id + self, user_id, version, room_id=None, session_id=None ): keyvalues = { @@ -159,10 +159,10 @@ def delete_e2e_room_keys( ) @defer.inlineCallbacks - def get_e2e_room_key_version_info(self, user_id, version): + def get_e2e_room_keys_version_info(self, user_id, version): row = yield self._simple_select_one( - table="e2e_room_key_versions", + table="e2e_room_keys_versions", keyvalues={ "user_id": user_id, "version": version, @@ -173,20 +173,20 @@ def get_e2e_room_key_version_info(self, user_id, version): "algorithm", "auth_data", ), - desc="get_e2e_room_key_version_info", + desc="get_e2e_room_keys_version_info", ) defer.returnValue(row) - def create_e2e_room_key_version(self, user_id, info): + def create_e2e_room_keys_version(self, user_id, info): """Atomically creates a new version of this user's e2e_room_keys store with the given version info. """ - def _create_e2e_room_key_version_txn(txn): + def _create_e2e_room_keys_version_txn(txn): txn.execute( - "SELECT MAX(version) FROM e2e_room_key_versions WHERE user_id=?", + "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?", (user_id,) ) current_version = txn.fetchone()[0] @@ -197,7 +197,7 @@ def _create_e2e_room_key_version_txn(txn): self._simple_insert_txn( txn, - table="e2e_room_key_versions", + table="e2e_room_keys_versions", values={ "user_id": user_id, "version": new_version, @@ -209,11 +209,11 @@ def _create_e2e_room_key_version_txn(txn): return new_version return self.runInteraction( - "create_e2e_room_key_version_txn", _create_e2e_room_key_version_txn + "create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn ) @defer.inlineCallbacks - def delete_e2e_room_key_version(self, user_id, version): + def delete_e2e_room_keys_version(self, user_id, version): keyvalues = { "user_id": user_id, @@ -221,7 +221,7 @@ def delete_e2e_room_key_version(self, user_id, version): } yield self._simple_delete( - table="e2e_room_key_versions", + table="e2e_room_keys_versions", keyvalues=keyvalues, - desc="delete_e2e_room_key_version", + desc="delete_e2e_room_keys_version", ) From ca0b052307de8868d3e337f1ace5667dad740ab1 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 24 Dec 2017 15:03:44 +0000 Subject: [PATCH 09/45] fix factoring out of _should_replace_room_key --- synapse/handlers/e2e_room_keys.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 87be081b1cfd..b67d6a2a7e68 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -100,15 +100,16 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): else: raise e - if _should_replace_room_key(current_room_key, room_key): + if E2eRoomKeysHandler._should_replace_room_key(current_room_key, room_key): yield self.store.set_e2e_room_key( user_id, version, room_id, session_id, room_key ) + @staticmethod def _should_replace_room_key(current_room_key, room_key): """ - Determine whether to replace the current_room_key in our backup for this - session (if any) with a new room_key that has been uploaded. + Determine whether to replace a given current_room_key (if any) + with a newly uploaded room_key backup Args: current_room_key (dict): Optional, the current room_key dict if any From 8d14598e90396c52c9394cf807861921e053d8da Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 24 Dec 2017 16:44:18 +0000 Subject: [PATCH 10/45] add storage docstring; remove unused set_e2e_room_keys --- synapse/storage/e2e_room_keys.py | 119 +++++++++++++++++++++---------- 1 file changed, 83 insertions(+), 36 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 7e1cb13e7427..45d2c9b4333c 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -22,6 +22,22 @@ class EndToEndRoomKeyStore(SQLBaseStore): @defer.inlineCallbacks def get_e2e_room_key(self, user_id, version, room_id, session_id): + """Get the encrypted E2E room key for a given session from a given + backup version of room_keys. We only store the 'best' room key for a given + session at a given time, as determined by the handler. + + Args: + user_id(str): the user whose backup we're querying + version(str): the version ID of the backup for the set of keys we're querying + room_id(str): the ID of the room whose keys we're querying. + This is a bit redundant as it's implied by the session_id, but + we include for consistency with the rest of the API. + session_id(str): the session whose room_key we're querying. + + Returns: + A deferred dict giving the session_data and message metadata for + this room key. + """ row = yield self._simple_select_one( table="e2e_room_keys", @@ -43,6 +59,17 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id): defer.returnValue(row) def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): + """Replaces or inserts the encrypted E2E room key for a given session in + a given backup + + Args: + user_id(str): the user whose backup we're setting + version(str): the version ID of the backup we're updating + room_id(str): the ID of the room whose keys we're setting + session_id(str): the session whose room_key we're setting + Raises: + StoreError if stuff goes wrong, probably + """ yield self._simple_upsert( table="e2e_room_keys", @@ -61,46 +88,27 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): lock=False, ) - # XXX: this isn't currently used and isn't tested anywhere - # it could be used in future for bulk-uploading new versions of room_keys - # for a user or something though. - def set_e2e_room_keys(self, user_id, version, room_keys): - - def _set_e2e_room_keys_txn(txn): - - values = [] - for room_id in room_keys['rooms']: - for session_id in room_keys['rooms'][room_id]['sessions']: - session = room_keys['rooms'][room_id]['sessions'][session_id] - values.append( - { - "user_id": user_id, - "room_id": room_id, - "session_id": session_id, - "version": version, - "first_message_index": session['first_message_index'], - "forwarded_count": session['forwarded_count'], - "is_verified": session['is_verified'], - "session_data": session['session_data'], - } - ) - - self._simple_insert_many_txn( - txn, - table="e2e_room_keys", - values=values - ) - - return True - - return self.runInteraction( - "set_e2e_room_keys", _set_e2e_room_keys_txn - ) - @defer.inlineCallbacks def get_e2e_room_keys( self, user_id, version, room_id=None, session_id=None ): + """Bulk get the E2E room keys for a given backup, optionally filtered to a given + room, or a given session. + + Args: + user_id(str): the user whose backup we're querying + version(str): the version ID of the backup for the set of keys we're querying + room_id(str): Optional. the ID of the room whose keys we're querying, if any. + If not specified, we return the keys for all the rooms in the backup. + session_id(str): Optional. the session whose room_key we're querying, if any. + If specified, we also require the room_id to be specified. + If not specified, we return all the keys in this version of + the backup (or for the specified room) + + Returns: + A deferred list of dicts giving the session_data and message metadata for + these room keys. + """ keyvalues = { "user_id": user_id, @@ -142,6 +150,22 @@ def get_e2e_room_keys( def delete_e2e_room_keys( self, user_id, version, room_id=None, session_id=None ): + """Bulk delete the E2E room keys for a given backup, optionally filtered to a given + room or a given session. + + Args: + user_id(str): the user whose backup we're deleting from + version(str): the version ID of the backup for the set of keys we're deleting + room_id(str): Optional. the ID of the room whose keys we're deleting, if any. + If not specified, we delete the keys for all the rooms in the backup. + session_id(str): Optional. the session whose room_key we're querying, if any. + If specified, we also require the room_id to be specified. + If not specified, we delete all the keys in this version of + the backup (or for the specified room) + + Returns: + A deferred of the deletion transaction + """ keyvalues = { "user_id": user_id, @@ -160,6 +184,15 @@ def delete_e2e_room_keys( @defer.inlineCallbacks def get_e2e_room_keys_version_info(self, user_id, version): + """Get info etadata about a given version of our room_keys backup + + Args: + user_id(str): the user whose backup we're querying + version(str): the version ID of the backup we're querying about + + Returns: + A deferred dict giving the info metadata for this backup version + """ row = yield self._simple_select_one( table="e2e_room_keys_versions", @@ -181,6 +214,13 @@ def get_e2e_room_keys_version_info(self, user_id, version): def create_e2e_room_keys_version(self, user_id, info): """Atomically creates a new version of this user's e2e_room_keys store with the given version info. + + Args: + user_id(str): the user whose backup we're creating a version + info(dict): the info about the backup version to be created + + Returns: + A deferred string for the newly created version ID """ def _create_e2e_room_keys_version_txn(txn): @@ -214,6 +254,13 @@ def _create_e2e_room_keys_version_txn(txn): @defer.inlineCallbacks def delete_e2e_room_keys_version(self, user_id, version): + """Delete a given backup version of the user's room keys. + Doesn't delete their actual key data. + + Args: + user_id(str): the user whose backup version we're deleting + version(str): the ID of the backup version we're deleting + """ keyvalues = { "user_id": user_id, From 9f500cb39efa608c02f212711a5ad2177757bf4b Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 24 Dec 2017 17:42:17 +0000 Subject: [PATCH 11/45] more docstring for the e2e_room_keys rest --- synapse/handlers/e2e_room_keys.py | 2 - synapse/rest/client/v2_alpha/room_keys.py | 51 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index b67d6a2a7e68..7a940d1c2132 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -59,7 +59,6 @@ def delete_room_keys(self, user_id, version, room_id, session_id): @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): - # TODO: Validate the JSON to make sure it has the right keys. # XXX: perhaps we should use a finer grained lock here? @@ -139,7 +138,6 @@ def _should_replace_room_key(current_room_key, room_key): @defer.inlineCallbacks def create_version(self, user_id, version_info): - # TODO: Validate the JSON to make sure it has the right keys. # lock everyone out until we've switched version diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 70b7b4573f57..04547c7d4371 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -208,6 +208,10 @@ def on_DELETE(self, request, room_id, session_id): """ Deletes one or more encrypted E2E room keys for a user for backup purposes. + DELETE /room_keys/keys/!abc:matrix.org/c0ff33?version=1 + HTTP/1.1 200 OK + {} + room_id: the ID of the room whose keys to delete (optional) session_id: the ID for the E2E session to delete (optional) version: the version of the user's backup which this data is for. @@ -240,6 +244,33 @@ def __init__(self, hs): @defer.inlineCallbacks def on_POST(self, request, version): + """ + Create a new backup version for this user's room_keys with the given + info. The version is allocated by the server and returned to the user + in the response. This API is intended to be used whenever the user + changes the encryption key for their backups, ensuring that backups + encrypted with different keys don't collide. + + The algorithm passed in the version info is a reverse-DNS namespaced + identifier to describe the format of the encrypted backupped keys. + + The auth_data is { user_id: "user_id", nonce: } + encrypted using the algorithm and current encryption key described above. + + POST /room_keys/version + Content-Type: application/json + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + + HTTP/1.1 200 OK + Content-Type: application/json + { + "version": 12345 + } + """ + if version: raise SynapseError(405, "Cannot POST to a specific version") @@ -257,6 +288,17 @@ def on_POST(self, request, version): @defer.inlineCallbacks def on_GET(self, request, version): + """ + Retrieve the version information about a given version of the user's + room_keys backup. + + GET /room_keys/version/12345 HTTP/1.1 + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() @@ -267,6 +309,15 @@ def on_GET(self, request, version): @defer.inlineCallbacks def on_DELETE(self, request, version): + """ + Delete the information about a given version of the user's + room_keys backup. Doesn't delete the actual room data. + + DELETE /room_keys/version/12345 HTTP/1.1 + HTTP/1.1 200 OK + {} + """ + requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() From 9f0791b7bd030a70182cd9b33bbe2f78dba705dd Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 27 Dec 2017 23:35:10 +0000 Subject: [PATCH 12/45] add a tonne of docstring; make upload_room_keys properly assert version --- synapse/storage/e2e_room_keys.py | 51 +++++++++++++++++++------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 45d2c9b4333c..e04e6a3690e1 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -67,6 +67,7 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): version(str): the version ID of the backup we're updating room_id(str): the ID of the room whose keys we're setting session_id(str): the session whose room_key we're setting + room_key(dict): the room_key being set Raises: StoreError if stuff goes wrong, probably """ @@ -182,35 +183,46 @@ def delete_e2e_room_keys( desc="delete_e2e_room_keys", ) - @defer.inlineCallbacks - def get_e2e_room_keys_version_info(self, user_id, version): - """Get info etadata about a given version of our room_keys backup + def get_e2e_room_keys_version_info(self, user_id, version=None): + """Get info metadata about a version of our room_keys backup. Args: user_id(str): the user whose backup we're querying - version(str): the version ID of the backup we're querying about - + version(str): Optional. the version ID of the backup we're querying about + If missing, we return the information about the current version. + Raises: + StoreError: with code 404 if there are no e2e_room_keys_versions present Returns: A deferred dict giving the info metadata for this backup version """ - row = yield self._simple_select_one( - table="e2e_room_keys_versions", - keyvalues={ - "user_id": user_id, - "version": version, - }, - retcols=( - "user_id", - "version", - "algorithm", - "auth_data", - ), + def _get_e2e_room_keys_version_info_txn(txn): + if version is None: + txn.execute( + "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?", + (user_id,) + ) + version = txn.fetchone()[0] + + return self._simple_select_one_txn( + table="e2e_room_keys_versions", + keyvalues={ + "user_id": user_id, + "version": version, + }, + retcols=( + "user_id", + "version", + "algorithm", + "auth_data", + ), + ) + + return self.runInteraction( desc="get_e2e_room_keys_version_info", + _get_e2e_room_keys_version_info_txn ) - defer.returnValue(row) - def create_e2e_room_keys_version(self, user_id, info): """Atomically creates a new version of this user's e2e_room_keys store with the given version info. @@ -224,7 +236,6 @@ def create_e2e_room_keys_version(self, user_id, info): """ def _create_e2e_room_keys_version_txn(txn): - txn.execute( "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?", (user_id,) From 14b3da63a339333292a83410c0ba3148bcb644ba Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 27 Dec 2017 23:37:44 +0000 Subject: [PATCH 13/45] add a tonne of docstring; make upload_room_keys properly assert version --- synapse/handlers/e2e_room_keys.py | 111 ++++++++++++++++++++-- synapse/rest/client/v2_alpha/room_keys.py | 11 ++- 2 files changed, 113 insertions(+), 9 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 7a940d1c2132..2fa025bfc74c 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -42,7 +42,16 @@ def __init__(self, hs): self._upload_linearizer = Linearizer("upload_room_keys_lock") @defer.inlineCallbacks - def get_room_keys(self, user_id, version, room_id, session_id): + def get_room_keys(self, user_id, version, room_id=None, session_id=None): + """Bulk get the E2E room keys for a given backup, optionally filtered to a given + room, or a given session. + See EndToEndRoomKeyStore.get_e2e_room_keys for full details. + + Returns: + A deferred list of dicts giving the session_data and message metadata for + these room keys. + """ + # we deliberately take the lock to get keys so that changing the version # works atomically with (yield self._upload_linearizer.queue(user_id)): @@ -52,20 +61,56 @@ def get_room_keys(self, user_id, version, room_id, session_id): defer.returnValue(results) @defer.inlineCallbacks - def delete_room_keys(self, user_id, version, room_id, session_id): + def delete_room_keys(self, user_id, version, room_id=None, session_id=None): + """Bulk delete the E2E room keys for a given backup, optionally filtered to a given + room or a given session. + See EndToEndRoomKeyStore.delete_e2e_room_keys for full details. + + Returns: + A deferred of the deletion transaction + """ + # lock for consistency with uploading with (yield self._upload_linearizer.queue(user_id)): yield self.store.delete_e2e_room_keys(user_id, version, room_id, session_id) @defer.inlineCallbacks def upload_room_keys(self, user_id, version, room_keys): + """Bulk upload a list of room keys into a given backup version, asserting + that the given version is the current backup version. room_keys are merged + into the current backup as described in RoomKeysServlet.on_PUT(). + + Args: + user_id(str): the user whose backup we're setting + version(str): the version ID of the backup we're updating + room_keys(dict): a nested dict describing the room_keys we're setting: + + { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": false, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } + } + + Raises: + SynapseError: with code 404 if there are no versions defined + RoomKeysVersionError: if the uploaded version is not the current version + """ + # TODO: Validate the JSON to make sure it has the right keys. # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): # Check that the version we're trying to upload is the current version - try: - version_info = yield self.get_version_info(user_id, version) + version_info = yield self.get_current_version_info(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) @@ -87,6 +132,17 @@ def upload_room_keys(self, user_id, version, room_keys): @defer.inlineCallbacks def _upload_room_key(self, user_id, version, room_id, session_id, room_key): + """Upload a given room_key for a given room and session into a given + version of the backup. Merges the key with any which might already exist. + + Args: + user_id(str): the user whose backup we're setting + version(str): the version ID of the backup we're updating + room_id(str): the ID of the room whose keys we're setting + session_id(str): the session whose room_key we're setting + room_key(dict): the room_key being set + """ + # get the room_key for this particular row current_room_key = None try: @@ -138,6 +194,23 @@ def _should_replace_room_key(current_room_key, room_key): @defer.inlineCallbacks def create_version(self, user_id, version_info): + """Create a new backup version. This automatically becomes the new + backup version for the user's keys; previous backups will no longer be + writeable to. + + Args: + user_id(str): the user whose backup version we're creating + version_info(dict): metadata about the new version being created + + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + + Returns: + A deferred of a string that gives the new version number. + """ + # TODO: Validate the JSON to make sure it has the right keys. # lock everyone out until we've switched version @@ -148,14 +221,36 @@ def create_version(self, user_id, version_info): defer.returnValue(new_version) @defer.inlineCallbacks - def get_version_info(self, user_id, version): + def get_current_version_info(self, user_id): + """Get the user's current backup version. + + Args: + user_id(str): the user whose current backup version we're querying + Raises: + StoreError: code 404 if there is no current backup version + Returns: + A deferred of a info dict that gives the info about the new version. + + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" + } + """ + with (yield self._upload_linearizer.queue(user_id)): - results = yield self.store.get_e2e_room_keys_version_info( - user_id, version - ) + results = yield self.store.get_e2e_room_keys_version_info(user_id) defer.returnValue(results) @defer.inlineCallbacks def delete_version(self, user_id, version): + """Deletes a given version of the user's e2e_room_keys backup + + Args: + user_id(str): the user whose current backup version we're deleting + version(str): the version id of the backup being deleted + Raises: + StoreError: code 404 if this backup version doesn't exist + """ + with (yield self._upload_linearizer.queue(user_id)): yield self.store.delete_e2e_room_keys_version(user_id, version) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 04547c7d4371..d3f857aba2e0 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -47,7 +47,7 @@ def on_PUT(self, request, room_id, session_id): room_id: the ID of the room the keys are for (optional) session_id: the ID for the E2E room keys for the room (optional) version: the version of the user's backup which this data is for. - the version must already have been created via the /change_secret API. + the version must already have been created via the /room_keys/version API. Each session has: * first_message_index: a numeric index indicating the oldest message @@ -59,6 +59,9 @@ def on_PUT(self, request, room_id, session_id): * session_data: base64-encrypted data describing the session. Returns 200 OK on success with body {} + Returns 403 Forbidden if the version in question is not the most recently + created version (i.e. if this is an old client trying to write to a stale backup) + Returns 404 Not Found if the version in question doesn't exist The API is designed to be otherwise agnostic to the room_key encryption algorithm being used. Sessions are merged with existing ones in the @@ -251,6 +254,9 @@ def on_POST(self, request, version): changes the encryption key for their backups, ensuring that backups encrypted with different keys don't collide. + It takes out an exclusive lock on this user's room_key backups, to ensure + clients only upload to the current backup. + The algorithm passed in the version info is a reverse-DNS namespaced identifier to describe the format of the encrypted backupped keys. @@ -292,6 +298,9 @@ def on_GET(self, request, version): Retrieve the version information about a given version of the user's room_keys backup. + It takes out an exclusive lock on this user's room_key backups, to ensure + clients only upload to the current backup. + GET /room_keys/version/12345 HTTP/1.1 { "algorithm": "m.megolm_backup.v1", From 234611f3472b229d9e3a9ec7a27c51446f6a61ba Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 27 Dec 2017 23:42:08 +0000 Subject: [PATCH 14/45] fix typos --- synapse/handlers/e2e_room_keys.py | 3 ++- synapse/storage/e2e_room_keys.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 2fa025bfc74c..6446c3c6c353 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -110,7 +110,8 @@ def upload_room_keys(self, user_id, version, room_keys): # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): # Check that the version we're trying to upload is the current version - version_info = yield self.get_current_version_info(user_id) + try: + version_info = yield self.get_current_version_info(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index e04e6a3690e1..b51faa1204e0 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -219,7 +219,7 @@ def _get_e2e_room_keys_version_info_txn(txn): ) return self.runInteraction( - desc="get_e2e_room_keys_version_info", + "get_e2e_room_keys_version_info", _get_e2e_room_keys_version_info_txn ) From 982edca38026e2bb9085e77b4c25af60c4793f34 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 27 Dec 2017 23:58:51 +0000 Subject: [PATCH 15/45] fix flakes --- synapse/handlers/e2e_room_keys.py | 4 ++-- synapse/storage/e2e_room_keys.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 6446c3c6c353..fdae69c60084 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -136,7 +136,7 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): """Upload a given room_key for a given room and session into a given version of the backup. Merges the key with any which might already exist. - Args: + Args: user_id(str): the user whose backup we're setting version(str): the version ID of the backup we're updating room_id(str): the ID of the room whose keys we're setting @@ -199,7 +199,7 @@ def create_version(self, user_id, version_info): backup version for the user's keys; previous backups will no longer be writeable to. - Args: + Args: user_id(str): the user whose backup version we're creating version_info(dict): metadata about the new version being created diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index b51faa1204e0..7ab75070a26f 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -202,13 +202,15 @@ def _get_e2e_room_keys_version_info_txn(txn): "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?", (user_id,) ) - version = txn.fetchone()[0] + this_version = txn.fetchone()[0] + else: + this_version = version return self._simple_select_one_txn( table="e2e_room_keys_versions", keyvalues={ "user_id": user_id, - "version": version, + "version": this_version, }, retcols=( "user_id", From 5e42c45c96bb62a86604118250e9cb6c57b94254 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 14:10:31 +0000 Subject: [PATCH 16/45] switch get_current_version_info back to being get_version_info --- synapse/handlers/e2e_room_keys.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index fdae69c60084..f08d80da3e5e 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -111,7 +111,7 @@ def upload_room_keys(self, user_id, version, room_keys): with (yield self._upload_linearizer.queue(user_id)): # Check that the version we're trying to upload is the current version try: - version_info = yield self.get_current_version_info(user_id) + version_info = yield self.get_version_info(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) @@ -222,17 +222,20 @@ def create_version(self, user_id, version_info): defer.returnValue(new_version) @defer.inlineCallbacks - def get_current_version_info(self, user_id): - """Get the user's current backup version. + def get_version_info(self, user_id, version=None): + """Get the info about a given version of the user's backup Args: user_id(str): the user whose current backup version we're querying + version(str): Optional; if None gives the most recent version + otherwise a historical one. Raises: - StoreError: code 404 if there is no current backup version + StoreError: code 404 if the requested backup version doesn't exist Returns: A deferred of a info dict that gives the info about the new version. { + "version": "1234", "algorithm": "m.megolm_backup.v1", "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" } From 93d174bcc4cf773bea2534d3683a1e91e5489c89 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 14:10:49 +0000 Subject: [PATCH 17/45] improve docstring --- synapse/rest/client/v2_alpha/room_keys.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index d3f857aba2e0..ca69ced1e31c 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -296,13 +296,17 @@ def on_POST(self, request, version): def on_GET(self, request, version): """ Retrieve the version information about a given version of the user's - room_keys backup. + room_keys backup. If the version part is missing, returns info about the + most current backup version (if any) It takes out an exclusive lock on this user's room_key backups, to ensure clients only upload to the current backup. + Returns 404 is the given version does not exist. + GET /room_keys/version/12345 HTTP/1.1 { + "version": "12345", "algorithm": "m.megolm_backup.v1", "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" } From b5eee511c73fdd9b5d1a7453433513791192a250 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 14:11:02 +0000 Subject: [PATCH 18/45] don't needlessly return user_id --- synapse/storage/e2e_room_keys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 7ab75070a26f..3c720f3b3e7c 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -213,7 +213,6 @@ def _get_e2e_room_keys_version_info_txn(txn): "version": this_version, }, retcols=( - "user_id", "version", "algorithm", "auth_data", From 174be586e5fca46013a4a8f03b4337f46b2502aa Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 14:11:15 +0000 Subject: [PATCH 19/45] first cut at a UT --- tests/handlers/test_e2e_room_keys.py | 141 +++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 tests/handlers/test_e2e_room_keys.py diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py new file mode 100644 index 000000000000..9d3bef6db2f1 --- /dev/null +++ b/tests/handlers/test_e2e_room_keys.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# Copyright 2017 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from synapse.api import errors +from twisted.internet import defer + +import synapse.api.errors +import synapse.handlers.e2e_room_keys + +import synapse.storage +from tests import unittest, utils + + +class E2eRoomKeysHandlerTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs): + super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs) + self.hs = None # type: synapse.server.HomeServer + self.handler = None # type: synapse.handlers.e2e_keys.E2eRoomKeysHandler + + @defer.inlineCallbacks + def setUp(self): + self.hs = yield utils.setup_test_homeserver( + handlers=None, + replication_layer=mock.Mock(), + ) + self.handler = synapse.handlers.e2e_keys.E2eRoomKeysHandler(self.hs) + + + @defer.inlineCallbacks + def test_get_missing_current_version_info(self): + """Check that we get a 404 if we ask for info about the current version + if there is no version. + """ + local_user = "@boris:" + self.hs.hostname + try: + res = yield self.handler.get_version_info(local_user); + except errors.SynapseError as e: + self.assertEqual(e.code, 404); + self.assertEqual(res, None); + + @defer.inlineCallbacks + def test_get_missing_version_info(self): + """Check that we get a 404 if we ask for info about a specific version + if it doesn't exist. + """ + local_user = "@boris:" + self.hs.hostname + try: + res = yield self.handler.get_version_info(local_user, "mrflibble"); + except errors.SynapseError as e: + self.assertEqual(e.code, 404); + self.assertEqual(res, None); + + @defer.inlineCallbacks + def test_create_version(self): + """Check that we can create and then retrieve versions. + """ + local_user = "@boris:" + self.hs.hostname + res = yield self.handler.create_version(user_id, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }); + self.assertEqual(res, "1"); + + # check we can retrieve it as the current version + res = yield self.handler.get_version_info(local_user); + self.assertDictEqual(res, { + "version": "1", + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }); + + # check we can retrieve it as a specific version + res = yield self.handler.get_version_info(local_user, "1"); + self.assertDictEqual(res, { + "version": "1", + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }); + + # upload a new one... + res = yield self.handler.create_version(user_id, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "second_version_auth_data", + }); + self.assertEqual(res, "2"); + + # check we can retrieve it as the current version + res = yield self.handler.get_version_info(local_user); + self.assertDictEqual(res, { + "version": "2", + "algorithm": "m.megolm_backup.v1", + "auth_data": "second_version_auth_data", + }); + + @defer.inlineCallbacks + def test_delete_version(self): + """Check that we can create and then delete versions. + """ + local_user = "@boris:" + self.hs.hostname + res = yield self.handler.create_version(user_id, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }); + self.assertEqual(res, "1"); + + # check we can delete it + yield self.handler.delete_version(local_user, "1"); + + # check that it's gone + try: + res = yield self.handler.get_version_info(local_user, "1"); + except errors.SynapseError as e: + self.assertEqual(e.code, 404); + self.assertEqual(res, None); + + + @defer.inlineCallbacks + def test_get_room_keys(self): + pass + + @defer.inlineCallbacks + def test_upload_room_keys(self): + pass + + @defer.inlineCallbacks + def test_delete_room_keys(self): + pass From 15d513f16fd272fb3763a42f05a8f296d4e1abf0 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 14:35:25 +0000 Subject: [PATCH 20/45] fix idiocies and so make tests pass --- synapse/storage/e2e_room_keys.py | 5 +++-- .../storage/schema/delta/46/e2e_room_keys.sql | 2 +- tests/handlers/test_e2e_room_keys.py | 19 +++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 3c720f3b3e7c..e4d56b7c3757 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -207,6 +207,7 @@ def _get_e2e_room_keys_version_info_txn(txn): this_version = version return self._simple_select_one_txn( + txn, table="e2e_room_keys_versions", keyvalues={ "user_id": user_id, @@ -243,9 +244,9 @@ def _create_e2e_room_keys_version_txn(txn): ) current_version = txn.fetchone()[0] if current_version is None: - current_version = 0 + current_version = '0' - new_version = current_version + 1 + new_version = str(int(current_version) + 1) self._simple_insert_txn( txn, diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/46/e2e_room_keys.sql index 16499ac34c23..4531fd56ee8a 100644 --- a/synapse/storage/schema/delta/46/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/46/e2e_room_keys.sql @@ -35,4 +35,4 @@ CREATE TABLE e2e_room_keys_versions ( auth_data TEXT NOT NULL ); -CREATE UNIQUE INDEX e2e_room_keys_versions_user_idx ON e2e_room_keys_versions(user_id); +CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version); diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 9d3bef6db2f1..6f4b3a147a03 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -37,7 +37,7 @@ def setUp(self): handlers=None, replication_layer=mock.Mock(), ) - self.handler = synapse.handlers.e2e_keys.E2eRoomKeysHandler(self.hs) + self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs) @defer.inlineCallbacks @@ -46,6 +46,7 @@ def test_get_missing_current_version_info(self): if there is no version. """ local_user = "@boris:" + self.hs.hostname + res = None try: res = yield self.handler.get_version_info(local_user); except errors.SynapseError as e: @@ -58,6 +59,7 @@ def test_get_missing_version_info(self): if it doesn't exist. """ local_user = "@boris:" + self.hs.hostname + res = None try: res = yield self.handler.get_version_info(local_user, "mrflibble"); except errors.SynapseError as e: @@ -69,7 +71,7 @@ def test_create_version(self): """Check that we can create and then retrieve versions. """ local_user = "@boris:" + self.hs.hostname - res = yield self.handler.create_version(user_id, { + res = yield self.handler.create_version(local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", }); @@ -92,7 +94,7 @@ def test_create_version(self): }); # upload a new one... - res = yield self.handler.create_version(user_id, { + res = yield self.handler.create_version(local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "second_version_auth_data", }); @@ -111,7 +113,7 @@ def test_delete_version(self): """Check that we can create and then delete versions. """ local_user = "@boris:" + self.hs.hostname - res = yield self.handler.create_version(user_id, { + res = yield self.handler.create_version(local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", }); @@ -121,21 +123,22 @@ def test_delete_version(self): yield self.handler.delete_version(local_user, "1"); # check that it's gone + res = None try: res = yield self.handler.get_version_info(local_user, "1"); except errors.SynapseError as e: self.assertEqual(e.code, 404); - self.assertEqual(res, None); + self.assertEqual(res, None); @defer.inlineCallbacks def test_get_room_keys(self): - pass + yield None @defer.inlineCallbacks def test_upload_room_keys(self): - pass + yield None @defer.inlineCallbacks def test_delete_room_keys(self): - pass + yield None From f6a3067868e7da2222484feb9aa421b1dcfecd0a Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 14:42:10 +0000 Subject: [PATCH 21/45] linting --- tests/handlers/test_e2e_room_keys.py | 48 +++++++++++++--------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 6f4b3a147a03..afe6ecf27bd4 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -39,7 +39,6 @@ def setUp(self): ) self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs) - @defer.inlineCallbacks def test_get_missing_current_version_info(self): """Check that we get a 404 if we ask for info about the current version @@ -48,10 +47,10 @@ def test_get_missing_current_version_info(self): local_user = "@boris:" + self.hs.hostname res = None try: - res = yield self.handler.get_version_info(local_user); + res = yield self.handler.get_version_info(local_user) except errors.SynapseError as e: - self.assertEqual(e.code, 404); - self.assertEqual(res, None); + self.assertEqual(e.code, 404) + self.assertEqual(res, None) @defer.inlineCallbacks def test_get_missing_version_info(self): @@ -61,10 +60,10 @@ def test_get_missing_version_info(self): local_user = "@boris:" + self.hs.hostname res = None try: - res = yield self.handler.get_version_info(local_user, "mrflibble"); + res = yield self.handler.get_version_info(local_user, "mrflibble") except errors.SynapseError as e: - self.assertEqual(e.code, 404); - self.assertEqual(res, None); + self.assertEqual(e.code, 404) + self.assertEqual(res, None) @defer.inlineCallbacks def test_create_version(self): @@ -74,39 +73,39 @@ def test_create_version(self): res = yield self.handler.create_version(local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", - }); - self.assertEqual(res, "1"); + }) + self.assertEqual(res, "1") # check we can retrieve it as the current version - res = yield self.handler.get_version_info(local_user); + res = yield self.handler.get_version_info(local_user) self.assertDictEqual(res, { "version": "1", "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", - }); + }) # check we can retrieve it as a specific version - res = yield self.handler.get_version_info(local_user, "1"); + res = yield self.handler.get_version_info(local_user, "1") self.assertDictEqual(res, { "version": "1", "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", - }); + }) # upload a new one... res = yield self.handler.create_version(local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "second_version_auth_data", - }); - self.assertEqual(res, "2"); + }) + self.assertEqual(res, "2") # check we can retrieve it as the current version - res = yield self.handler.get_version_info(local_user); + res = yield self.handler.get_version_info(local_user) self.assertDictEqual(res, { "version": "2", "algorithm": "m.megolm_backup.v1", "auth_data": "second_version_auth_data", - }); + }) @defer.inlineCallbacks def test_delete_version(self): @@ -116,20 +115,19 @@ def test_delete_version(self): res = yield self.handler.create_version(local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", - }); - self.assertEqual(res, "1"); + }) + self.assertEqual(res, "1") # check we can delete it - yield self.handler.delete_version(local_user, "1"); + yield self.handler.delete_version(local_user, "1") # check that it's gone - res = None + res = None try: - res = yield self.handler.get_version_info(local_user, "1"); + res = yield self.handler.get_version_info(local_user, "1") except errors.SynapseError as e: - self.assertEqual(e.code, 404); - self.assertEqual(res, None); - + self.assertEqual(e.code, 404) + self.assertEqual(res, None) @defer.inlineCallbacks def test_get_room_keys(self): From fe87890b18f57f0268bd65aeca881e7817bbe9e4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 17:47:11 +0000 Subject: [PATCH 22/45] implement remaining tests and make them work --- synapse/handlers/e2e_room_keys.py | 35 ++- synapse/rest/client/v2_alpha/room_keys.py | 6 + synapse/storage/e2e_room_keys.py | 3 +- tests/handlers/test_e2e_room_keys.py | 276 ++++++++++++++++++++-- 4 files changed, 287 insertions(+), 33 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index f08d80da3e5e..09c2888db638 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -58,6 +58,10 @@ def get_room_keys(self, user_id, version, room_id=None, session_id=None): results = yield self.store.get_e2e_room_keys( user_id, version, room_id, session_id ) + + if results['rooms'] == {}: + raise SynapseError(404, "No room_keys found") + defer.returnValue(results) @defer.inlineCallbacks @@ -109,9 +113,10 @@ def upload_room_keys(self, user_id, version, room_keys): # XXX: perhaps we should use a finer grained lock here? with (yield self._upload_linearizer.queue(user_id)): + # Check that the version we're trying to upload is the current version try: - version_info = yield self.get_version_info(user_id) + version_info = yield self._get_version_info_unlocked(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) @@ -119,16 +124,23 @@ def upload_room_keys(self, user_id, version, room_keys): raise e if version_info['version'] != version: - raise RoomKeysVersionError(current_version=version_info.version) + # Check that the version we're trying to upload actually exists + try: + version_info = yield self._get_version_info_unlocked(user_id, version) + # if we get this far, the version must exist + raise RoomKeysVersionError(current_version=version_info['version']) + except StoreError as e: + if e.code == 404: + raise SynapseError(404, "Version '%s' not found" % (version,)) + else: + raise e # go through the room_keys. # XXX: this should/could be done concurrently, given we're in a lock. for room_id, room in room_keys['rooms'].iteritems(): for session_id, session in room['sessions'].iteritems(): - room_key = session[session_id] - yield self._upload_room_key( - user_id, version, room_id, session_id, room_key + user_id, version, room_id, session_id, session ) @defer.inlineCallbacks @@ -242,8 +254,17 @@ def get_version_info(self, user_id, version=None): """ with (yield self._upload_linearizer.queue(user_id)): - results = yield self.store.get_e2e_room_keys_version_info(user_id) - defer.returnValue(results) + res = yield self._get_version_info_unlocked(user_id, version) + defer.returnValue(res) + + @defer.inlineCallbacks + def _get_version_info_unlocked(self, user_id, version=None): + """Get the info about a given version of the user's backup + without obtaining the upload_linearizer lock. For params see get_version_info + """ + + results = yield self.store.get_e2e_room_keys_version_info(user_id, version) + defer.returnValue(results) @defer.inlineCallbacks def delete_version(self, user_id, version): diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index ca69ced1e31c..8f10e4e1cd72 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -204,6 +204,12 @@ def on_GET(self, request, room_id, session_id): room_keys = yield self.e2e_room_keys_handler.get_room_keys( user_id, version, room_id, session_id ) + + if session_id: + room_keys = room_keys['rooms'][room_id]['sessions'][session_id] + elif room_id: + room_keys = room_keys['rooms'][room_id] + defer.returnValue((200, room_keys)) @defer.inlineCallbacks diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index e4d56b7c3757..8e8e4e457ce4 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -58,6 +58,7 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id): defer.returnValue(row) + @defer.inlineCallbacks def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): """Replaces or inserts the encrypted E2E room key for a given session in a given backup @@ -135,7 +136,7 @@ def get_e2e_room_keys( desc="get_e2e_room_keys", ) - sessions = {} + sessions = { 'rooms': {} } for row in rows: room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}}) room_entry['sessions'][row['session_id']] = { diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index afe6ecf27bd4..3cbfd6f9d0c2 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -17,6 +17,7 @@ import mock from synapse.api import errors from twisted.internet import defer +import copy import synapse.api.errors import synapse.handlers.e2e_room_keys @@ -25,6 +26,22 @@ from tests import unittest, utils +# sample room_key data for use in the tests +room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK" + } + } + } + } +} + class E2eRoomKeysHandlerTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs) @@ -38,46 +55,44 @@ def setUp(self): replication_layer=mock.Mock(), ) self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs) + self.local_user = "@boris:" + self.hs.hostname; @defer.inlineCallbacks def test_get_missing_current_version_info(self): """Check that we get a 404 if we ask for info about the current version if there is no version. """ - local_user = "@boris:" + self.hs.hostname res = None try: - res = yield self.handler.get_version_info(local_user) + yield self.handler.get_version_info(self.local_user) except errors.SynapseError as e: - self.assertEqual(e.code, 404) - self.assertEqual(res, None) + res = e.code + self.assertEqual(res, 404) @defer.inlineCallbacks def test_get_missing_version_info(self): """Check that we get a 404 if we ask for info about a specific version if it doesn't exist. """ - local_user = "@boris:" + self.hs.hostname res = None try: - res = yield self.handler.get_version_info(local_user, "mrflibble") + yield self.handler.get_version_info(self.local_user, "bogus_version") except errors.SynapseError as e: - self.assertEqual(e.code, 404) - self.assertEqual(res, None) + res = e.code + self.assertEqual(res, 404) @defer.inlineCallbacks def test_create_version(self): """Check that we can create and then retrieve versions. """ - local_user = "@boris:" + self.hs.hostname - res = yield self.handler.create_version(local_user, { + res = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", }) self.assertEqual(res, "1") # check we can retrieve it as the current version - res = yield self.handler.get_version_info(local_user) + res = yield self.handler.get_version_info(self.local_user) self.assertDictEqual(res, { "version": "1", "algorithm": "m.megolm_backup.v1", @@ -85,7 +100,7 @@ def test_create_version(self): }) # check we can retrieve it as a specific version - res = yield self.handler.get_version_info(local_user, "1") + res = yield self.handler.get_version_info(self.local_user, "1") self.assertDictEqual(res, { "version": "1", "algorithm": "m.megolm_backup.v1", @@ -93,14 +108,14 @@ def test_create_version(self): }) # upload a new one... - res = yield self.handler.create_version(local_user, { + res = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "second_version_auth_data", }) self.assertEqual(res, "2") # check we can retrieve it as the current version - res = yield self.handler.get_version_info(local_user) + res = yield self.handler.get_version_info(self.local_user) self.assertDictEqual(res, { "version": "2", "algorithm": "m.megolm_backup.v1", @@ -111,32 +126,243 @@ def test_create_version(self): def test_delete_version(self): """Check that we can create and then delete versions. """ - local_user = "@boris:" + self.hs.hostname - res = yield self.handler.create_version(local_user, { + res = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", }) self.assertEqual(res, "1") # check we can delete it - yield self.handler.delete_version(local_user, "1") + yield self.handler.delete_version(self.local_user, "1") # check that it's gone res = None try: - res = yield self.handler.get_version_info(local_user, "1") + yield self.handler.get_version_info(self.local_user, "1") except errors.SynapseError as e: - self.assertEqual(e.code, 404) - self.assertEqual(res, None) + res = e.code + self.assertEqual(res, 404) @defer.inlineCallbacks - def test_get_room_keys(self): - yield None + def test_get_missing_room_keys(self): + """Check that we get a 404 on querying missing room_keys + """ + res = None + try: + yield self.handler.get_room_keys(self.local_user, "bogus_version") + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # check we also get a 404 even if the version is valid + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = None + try: + yield self.handler.get_room_keys(self.local_user, version) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # TODO: test the locking semantics when uploading room_keys, + # although this is probably best done in sytest @defer.inlineCallbacks - def test_upload_room_keys(self): - yield None + def test_upload_room_keys_no_versions(self): + """Check that we get a 404 on uploading keys when no versions are defined + """ + res = None + try: + yield self.handler.upload_room_keys(self.local_user, "no_version", room_keys) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_upload_room_keys_bogus_version(self): + """Check that we get a 404 on uploading keys when an nonexistent version is specified + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + res = None + try: + yield self.handler.upload_room_keys(self.local_user, "bogus_version", room_keys) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_upload_room_keys_wrong_version(self): + """Check that we get a 403 on uploading keys for an old version + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "second_version_auth_data", + }) + self.assertEqual(version, "2") + + res = None + try: + yield self.handler.upload_room_keys(self.local_user, "1", room_keys) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 403) + + @defer.inlineCallbacks + def test_upload_room_keys_insert(self): + """Check that we can insert and retrieve keys for a session + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertDictEqual(res, room_keys) + + # check getting room_keys for a given room + res = yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org" + ) + self.assertDictEqual(res, room_keys) + + # check getting room_keys for a given session_id + res = yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + self.assertDictEqual(res, room_keys) + + @defer.inlineCallbacks + def test_upload_room_keys_merge(self): + """Check that we can upload a new room_key for an existing session and + have it correctly merged""" + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + + new_room_keys = copy.deepcopy(room_keys) + + # test that increasing the message_index doesn't replace the existing session + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['first_message_index'] = 2 + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'new' + yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertEqual( + res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], + "SSBBTSBBIEZJU0gK" + ) + + # test that marking the session as verified however /does/ replace it + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['is_verified'] = True + yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertEqual( + res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], + "new" + ) + + # test that a session with a higher forwarded_count doesn't replace one + # with a lower forwarding count + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['forwarded_count'] = 2 + new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'other' + yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) + + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertEqual( + res['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'], + "new" + ) + + # TODO: check edge cases as well as the common variations here @defer.inlineCallbacks def test_delete_room_keys(self): - yield None + """Check that we can insert and delete keys for a session + """ + version = yield self.handler.create_version(self.local_user, { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }) + self.assertEqual(version, "1") + + # check for bulk-delete + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + yield self.handler.delete_room_keys(self.local_user, version) + res = None + try: + yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # check for bulk-delete per room + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + yield self.handler.delete_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + ) + res = None + try: + yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + # check for bulk-delete per session + yield self.handler.upload_room_keys(self.local_user, version, room_keys) + yield self.handler.delete_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + res = None + try: + yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) From edc427a35187e462458d188039e761ffdb7e486d Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 31 Dec 2017 17:50:55 +0000 Subject: [PATCH 23/45] flake8 --- synapse/storage/e2e_room_keys.py | 2 +- tests/handlers/test_e2e_room_keys.py | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 8e8e4e457ce4..c2f226396d51 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -136,7 +136,7 @@ def get_e2e_room_keys( desc="get_e2e_room_keys", ) - sessions = { 'rooms': {} } + sessions = {'rooms': {}} for row in rows: room_entry = sessions['rooms'].setdefault(row['room_id'], {"sessions": {}}) room_entry['sessions'][row['session_id']] = { diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 3cbfd6f9d0c2..6e43543ed96d 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -42,6 +42,7 @@ } } + class E2eRoomKeysHandlerTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): super(E2eRoomKeysHandlerTestCase, self).__init__(*args, **kwargs) @@ -55,7 +56,7 @@ def setUp(self): replication_layer=mock.Mock(), ) self.handler = synapse.handlers.e2e_room_keys.E2eRoomKeysHandler(self.hs) - self.local_user = "@boris:" + self.hs.hostname; + self.local_user = "@boris:" + self.hs.hostname @defer.inlineCallbacks def test_get_missing_current_version_info(self): @@ -184,7 +185,8 @@ def test_upload_room_keys_no_versions(self): @defer.inlineCallbacks def test_upload_room_keys_bogus_version(self): - """Check that we get a 404 on uploading keys when an nonexistent version is specified + """Check that we get a 404 on uploading keys when an nonexistent version + is specified """ version = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", @@ -194,7 +196,9 @@ def test_upload_room_keys_bogus_version(self): res = None try: - yield self.handler.upload_room_keys(self.local_user, "bogus_version", room_keys) + yield self.handler.upload_room_keys( + self.local_user, "bogus_version", room_keys + ) except errors.SynapseError as e: res = e.code self.assertEqual(res, 404) @@ -267,10 +271,11 @@ def test_upload_room_keys_merge(self): yield self.handler.upload_room_keys(self.local_user, version, room_keys) new_room_keys = copy.deepcopy(room_keys) + new_room_key = new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33'] # test that increasing the message_index doesn't replace the existing session - new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['first_message_index'] = 2 - new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'new' + new_room_key['first_message_index'] = 2 + new_room_key['session_data'] = 'new' yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) res = yield self.handler.get_room_keys(self.local_user, version) @@ -280,7 +285,7 @@ def test_upload_room_keys_merge(self): ) # test that marking the session as verified however /does/ replace it - new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['is_verified'] = True + new_room_key['is_verified'] = True yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) res = yield self.handler.get_room_keys(self.local_user, version) @@ -291,8 +296,8 @@ def test_upload_room_keys_merge(self): # test that a session with a higher forwarded_count doesn't replace one # with a lower forwarding count - new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['forwarded_count'] = 2 - new_room_keys['rooms']['!abc:matrix.org']['sessions']['c0ff33']['session_data'] = 'other' + new_room_key['forwarded_count'] = 2 + new_room_key['session_data'] = 'other' yield self.handler.upload_room_keys(self.local_user, version, new_room_keys) res = yield self.handler.get_room_keys(self.local_user, version) From 72788cf9c1f018a5346a8f9204c6bfe058289fd2 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:45:26 +0000 Subject: [PATCH 24/45] support DELETE /version with no args --- synapse/handlers/e2e_room_keys.py | 2 +- synapse/rest/client/v2_alpha/room_keys.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 09c2888db638..a43fc7fc7e5b 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -267,7 +267,7 @@ def _get_version_info_unlocked(self, user_id, version=None): defer.returnValue(results) @defer.inlineCallbacks - def delete_version(self, user_id, version): + def delete_version(self, user_id, version=None): """Deletes a given version of the user's e2e_room_keys backup Args: diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 8f10e4e1cd72..63b1f62f90e3 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -308,7 +308,7 @@ def on_GET(self, request, version): It takes out an exclusive lock on this user's room_key backups, to ensure clients only upload to the current backup. - Returns 404 is the given version does not exist. + Returns 404 if the given version does not exist. GET /room_keys/version/12345 HTTP/1.1 { @@ -330,7 +330,8 @@ def on_GET(self, request, version): def on_DELETE(self, request, version): """ Delete the information about a given version of the user's - room_keys backup. Doesn't delete the actual room data. + room_keys backup. If the version part is missing, deletes the most + current backup version (if any). Doesn't delete the actual room data. DELETE /room_keys/version/12345 HTTP/1.1 HTTP/1.1 200 OK From 66a4ca1d28c2c2e22a0343e6db0f5a2bce9ec987 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:45:55 +0000 Subject: [PATCH 25/45] 404 nicely if you try to interact with a missing current version --- synapse/storage/e2e_room_keys.py | 51 +++++++++++++++++++--------- tests/handlers/test_e2e_room_keys.py | 22 ++++++++++++ 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index c2f226396d51..d82f223e8665 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -184,6 +184,17 @@ def delete_e2e_room_keys( desc="delete_e2e_room_keys", ) + @staticmethod + def _get_current_version(txn, user_id): + txn.execute( + "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?", + (user_id,) + ) + row = txn.fetchone() + if not row: + raise StoreError(404, 'No current backup version') + return row[0] + def get_e2e_room_keys_version_info(self, user_id, version=None): """Get info metadata about a version of our room_keys backup. @@ -199,11 +210,7 @@ def get_e2e_room_keys_version_info(self, user_id, version=None): def _get_e2e_room_keys_version_info_txn(txn): if version is None: - txn.execute( - "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?", - (user_id,) - ) - this_version = txn.fetchone()[0] + this_version = self._get_current_version(txn, user_id) else: this_version = version @@ -266,23 +273,35 @@ def _create_e2e_room_keys_version_txn(txn): "create_e2e_room_keys_version_txn", _create_e2e_room_keys_version_txn ) - @defer.inlineCallbacks - def delete_e2e_room_keys_version(self, user_id, version): + def delete_e2e_room_keys_version(self, user_id, version=None): """Delete a given backup version of the user's room keys. Doesn't delete their actual key data. Args: user_id(str): the user whose backup version we're deleting - version(str): the ID of the backup version we're deleting + version(str): Optional. the version ID of the backup version we're deleting + If missing, we delete the current backup version info. + Raises: + StoreError: with code 404 if there are no e2e_room_keys_versions present, + or if the version requested doesn't exist. """ - keyvalues = { - "user_id": user_id, - "version": version, - } + def _delete_e2e_room_keys_version_txn(txn): + if version is None: + this_version = self._get_current_version(txn, user_id) + else: + this_version = version - yield self._simple_delete( - table="e2e_room_keys_versions", - keyvalues=keyvalues, - desc="delete_e2e_room_keys_version", + return self._simple_delete_one_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={ + "user_id": user_id, + "version": this_version, + }, + ) + + return self.runInteraction( + "delete_e2e_room_keys_version", + _delete_e2e_room_keys_version_txn ) diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 6e43543ed96d..8bfffb5c0e51 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -123,6 +123,28 @@ def test_create_version(self): "auth_data": "second_version_auth_data", }) + @defer.inlineCallbacks + def test_delete_missing_version(self): + """Check that we get a 404 on deleting nonexistent versions + """ + res = None + try: + yield self.handler.delete_version(self.local_user, "1") + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + + @defer.inlineCallbacks + def test_delete_missing_current_version(self): + """Check that we get a 404 on deleting nonexistent current version + """ + res = None + try: + yield self.handler.delete_version(self.local_user) + except errors.SynapseError as e: + res = e.code + self.assertEqual(res, 404) + @defer.inlineCallbacks def test_delete_version(self): """Check that we can create and then delete versions. From 54ac18e8327e689b9f91b93ad8caac2f6b3dd29c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:48:22 +0000 Subject: [PATCH 26/45] use parse_string --- synapse/rest/client/v2_alpha/room_keys.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 63b1f62f90e3..afb8a36dd0df 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -123,7 +123,7 @@ def on_PUT(self, request, room_id, session_id): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() body = parse_json_object_from_request(request) - version = request.args.get("version")[0] + version = parse_string(request, "version") if session_id: body = { @@ -199,7 +199,7 @@ def on_GET(self, request, room_id, session_id): """ requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version")[0] + version = parse_string(request, "version") room_keys = yield self.e2e_room_keys_handler.get_room_keys( user_id, version, room_id, session_id @@ -229,7 +229,7 @@ def on_DELETE(self, request, room_id, session_id): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - version = request.args.get("version")[0] + version = parse_string(request, "version") yield self.e2e_room_keys_handler.delete_room_keys( user_id, version, room_id, session_id From f0cede5556a54ece6cd5289856bb230f8009c4fa Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:58:32 +0000 Subject: [PATCH 27/45] missing import --- synapse/storage/e2e_room_keys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index d82f223e8665..089989fcfa5f 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -14,6 +14,7 @@ # limitations under the License. from twisted.internet import defer +from synapse.api.errors import StoreError from ._base import SQLBaseStore From 4f7064f6b5a1822067bfbf358317bc1dbb51b9c6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sun, 7 Jan 2018 23:59:07 +0000 Subject: [PATCH 28/45] missing import --- synapse/rest/client/v2_alpha/room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index afb8a36dd0df..9f0172e6f5ee 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -19,7 +19,7 @@ from synapse.api.errors import SynapseError from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request + RestServlet, parse_json_object_from_request, parse_string ) from ._base import client_v2_patterns From 8550a7e9c2fcf9e46c717127d14c79010350a998 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 21 Aug 2018 10:38:00 -0400 Subject: [PATCH 29/45] allow auth_data to be any JSON instead of a string --- synapse/storage/e2e_room_keys.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 089989fcfa5f..c7b1fad21e79 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -15,6 +15,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError +import simplejson as json from ._base import SQLBaseStore @@ -215,7 +216,7 @@ def _get_e2e_room_keys_version_info_txn(txn): else: this_version = version - return self._simple_select_one_txn( + result = self._simple_select_one_txn( txn, table="e2e_room_keys_versions", keyvalues={ @@ -228,6 +229,8 @@ def _get_e2e_room_keys_version_info_txn(txn): "auth_data", ), ) + result["auth_data"] = json.loads(result["auth_data"]) + return result return self.runInteraction( "get_e2e_room_keys_version_info", @@ -264,7 +267,7 @@ def _create_e2e_room_keys_version_txn(txn): "user_id": user_id, "version": new_version, "algorithm": info["algorithm"], - "auth_data": info["auth_data"], + "auth_data": json.dumps(info["auth_data"]), }, ) From 42a394caa2884096e5afe1f1c2c75680792769d8 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Tue, 21 Aug 2018 14:51:34 -0400 Subject: [PATCH 30/45] allow session_data to be any JSON instead of just a string --- synapse/storage/e2e_room_keys.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index c7b1fad21e79..b695570a7b70 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -58,6 +58,8 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id): desc="get_e2e_room_key", ) + row["session_data"] = json.loads(row["session_data"]); + defer.returnValue(row) @defer.inlineCallbacks @@ -87,7 +89,7 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): "first_message_index": room_key['first_message_index'], "forwarded_count": room_key['forwarded_count'], "is_verified": room_key['is_verified'], - "session_data": room_key['session_data'], + "session_data": json.dumps(room_key['session_data']), }, lock=False, ) @@ -145,7 +147,7 @@ def get_e2e_room_keys( "first_message_index": row["first_message_index"], "forwarded_count": row["forwarded_count"], "is_verified": row["is_verified"], - "session_data": row["session_data"], + "session_data": json.loads(row["session_data"]), } defer.returnValue(sessions) From 16a31c6fcebe31bbdb655ebcb24ccce426b3e994 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 24 Aug 2018 22:51:25 -0400 Subject: [PATCH 31/45] update to newer Synapse APIs --- synapse/handlers/e2e_room_keys.py | 2 +- tests/handlers/test_e2e_room_keys.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index a43fc7fc7e5b..c09816b37274 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -18,7 +18,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError -from synapse.util.async import Linearizer +from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 8bfffb5c0e51..7fa426444152 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -52,6 +52,7 @@ def __init__(self, *args, **kwargs): @defer.inlineCallbacks def setUp(self): self.hs = yield utils.setup_test_homeserver( + self.addCleanup, handlers=None, replication_layer=mock.Mock(), ) From 3801b8aa035594972c400c8bd036894a388c4ab3 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 6 Sep 2018 11:23:16 -0400 Subject: [PATCH 32/45] try to make flake8 and isort happy --- synapse/api/errors.py | 1 + synapse/handlers/e2e_room_keys.py | 2 +- synapse/rest/client/v2_alpha/room_keys.py | 5 ++++- synapse/storage/e2e_room_keys.py | 6 ++++-- tests/handlers/test_e2e_room_keys.py | 9 +++++---- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 3002c95dd1cd..140dbfe8b864 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -326,6 +326,7 @@ def __init__(self, current_version): ) self.current_version = current_version + class IncompatibleRoomVersionError(SynapseError): """A server is trying to join a room whose version it does not support.""" diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index c09816b37274..2c330382cfc6 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -17,7 +17,7 @@ from twisted.internet import defer -from synapse.api.errors import StoreError, SynapseError, RoomKeysVersionError +from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 9f0172e6f5ee..1ed18e986fa4 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -19,8 +19,11 @@ from synapse.api.errors import SynapseError from synapse.http.servlet import ( - RestServlet, parse_json_object_from_request, parse_string + RestServlet, + parse_json_object_from_request, + parse_string, ) + from ._base import client_v2_patterns logger = logging.getLogger(__name__) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index b695570a7b70..969f4aef9ca3 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import simplejson as json + from twisted.internet import defer + from synapse.api.errors import StoreError -import simplejson as json from ._base import SQLBaseStore @@ -58,7 +60,7 @@ def get_e2e_room_key(self, user_id, version, room_id, session_id): desc="get_e2e_room_key", ) - row["session_data"] = json.loads(row["session_data"]); + row["session_data"] = json.loads(row["session_data"]) defer.returnValue(row) diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 7fa426444152..9e08eac0a5bf 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -14,17 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy + import mock -from synapse.api import errors + from twisted.internet import defer -import copy import synapse.api.errors import synapse.handlers.e2e_room_keys - import synapse.storage -from tests import unittest, utils +from synapse.api import errors +from tests import unittest, utils # sample room_key data for use in the tests room_keys = { From bc74925c5b94f0c02603297d4ccb7e05008d5124 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 13 Sep 2018 17:02:59 +0100 Subject: [PATCH 33/45] WIP e2e key backups Continues from uhoreg's branch This just fixed the errcode on /room_keys/version if no backup and updates the schema delta to be on the latest so it gets run --- synapse/rest/client/v2_alpha/room_keys.py | 14 ++++++++++---- .../schema/delta/{46 => 51}/e2e_room_keys.sql | 0 2 files changed, 10 insertions(+), 4 deletions(-) rename synapse/storage/schema/delta/{46 => 51}/e2e_room_keys.sql (100%) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 1ed18e986fa4..ea114bc8b48e 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,7 +17,7 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError +from synapse.api.errors import SynapseError, Codes from synapse.http.servlet import ( RestServlet, parse_json_object_from_request, @@ -324,9 +324,15 @@ def on_GET(self, request, version): requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() - info = yield self.e2e_room_keys_handler.get_version_info( - user_id, version - ) + try: + info = yield self.e2e_room_keys_handler.get_version_info( + user_id, version + ) + except SynapseError as e: + if e.code == 404: + e.errcode = Codes.NOT_FOUND + e.msg = "No backup found" + raise e defer.returnValue((200, info)) @defer.inlineCallbacks diff --git a/synapse/storage/schema/delta/46/e2e_room_keys.sql b/synapse/storage/schema/delta/51/e2e_room_keys.sql similarity index 100% rename from synapse/storage/schema/delta/46/e2e_room_keys.sql rename to synapse/storage/schema/delta/51/e2e_room_keys.sql From 497444f1fdd2c39906179b1dde8c67415e465398 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 5 Oct 2018 15:08:36 +0100 Subject: [PATCH 34/45] Don't reuse backup versions Since we don't actually delete the keys, just mark the versions as deleted in the db rather than actually deleting them, then we won't reuse versions. Fixes https://github.com/vector-im/riot-web/issues/7448 --- synapse/storage/e2e_room_keys.py | 9 +++++++-- synapse/storage/schema/delta/51/e2e_room_keys.sql | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 969f4aef9ca3..4d439bb16473 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -193,7 +193,8 @@ def delete_e2e_room_keys( @staticmethod def _get_current_version(txn, user_id): txn.execute( - "SELECT MAX(version) FROM e2e_room_keys_versions WHERE user_id=?", + "SELECT MAX(version) FROM e2e_room_keys_versions " + "WHERE user_id=? AND deleted=0", (user_id,) ) row = txn.fetchone() @@ -226,6 +227,7 @@ def _get_e2e_room_keys_version_info_txn(txn): keyvalues={ "user_id": user_id, "version": this_version, + "deleted": 0, }, retcols=( "version", @@ -300,13 +302,16 @@ def _delete_e2e_room_keys_version_txn(txn): else: this_version = version - return self._simple_delete_one_txn( + return self._simple_update_one_txn( txn, table="e2e_room_keys_versions", keyvalues={ "user_id": user_id, "version": this_version, }, + updatevalues={ + "deleted": 1, + } ) return self.runInteraction( diff --git a/synapse/storage/schema/delta/51/e2e_room_keys.sql b/synapse/storage/schema/delta/51/e2e_room_keys.sql index 4531fd56ee8a..c0e66a697d88 100644 --- a/synapse/storage/schema/delta/51/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/51/e2e_room_keys.sql @@ -32,7 +32,8 @@ CREATE TABLE e2e_room_keys_versions ( user_id TEXT NOT NULL, version TEXT NOT NULL, algorithm TEXT NOT NULL, - auth_data TEXT NOT NULL + auth_data TEXT NOT NULL, + deleted SMALLINT DEFAULT 0 NOT NULL ); CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version); From 0c905ee015fdbf0d61f46db600fd79a18d0b6376 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 9 Oct 2018 09:39:13 +0100 Subject: [PATCH 35/45] be python3 compatible --- synapse/handlers/e2e_room_keys.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 2c330382cfc6..117144bb6eb3 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -15,6 +15,7 @@ import logging +from six import iteritems from twisted.internet import defer from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError @@ -137,8 +138,8 @@ def upload_room_keys(self, user_id, version, room_keys): # go through the room_keys. # XXX: this should/could be done concurrently, given we're in a lock. - for room_id, room in room_keys['rooms'].iteritems(): - for session_id, session in room['sessions'].iteritems(): + for room_id, room in iteritems(room_keys['rooms']): + for session_id, session in iteritems(room['sessions']): yield self._upload_room_key( user_id, version, room_id, session_id, session ) From f4a4dbcad106b41e3307eb5ca541ff3087b788cf Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 9 Oct 2018 09:47:04 +0100 Subject: [PATCH 36/45] Apparently this blank line is Very Important --- synapse/handlers/e2e_room_keys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 117144bb6eb3..bf2a83cc31c9 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -16,6 +16,7 @@ import logging from six import iteritems + from twisted.internet import defer from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError From d3464ce708cd857216d88d3e67867f1cf0c8bf60 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 9 Oct 2018 10:33:59 +0100 Subject: [PATCH 37/45] isort --- synapse/rest/client/v2_alpha/room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index ea114bc8b48e..539893a5d64b 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,7 +17,7 @@ from twisted.internet import defer -from synapse.api.errors import SynapseError, Codes +from synapse.api.errors import Codes, SynapseError from synapse.http.servlet import ( RestServlet, parse_json_object_from_request, From d34657e1f27df042a33cee4ba8c0b7cbaa112186 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 9 Oct 2018 11:15:54 +0100 Subject: [PATCH 38/45] Add changelog --- changelog.d/4019.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4019.feature diff --git a/changelog.d/4019.feature b/changelog.d/4019.feature new file mode 100644 index 000000000000..49e066d269ef --- /dev/null +++ b/changelog.d/4019.feature @@ -0,0 +1 @@ +Add support for end-to-end key backup (MSC1687) From b8d9e108be60b3d14a57238562bfb5f49781c2a6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 9 Oct 2018 18:04:21 +0100 Subject: [PATCH 39/45] Fix mergefail --- synapse/api/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 4124469442af..0a6e78711f29 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -344,7 +344,7 @@ def error_dict(self): return cs_error( self.msg, self.errcode, - room_version=self.current_version, + room_version=self._room_version, ) From 83e72bb2f0c6ef282190a378941c856afbb33c16 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 11:26:18 +0100 Subject: [PATCH 40/45] PR feedback pt. 1 --- synapse/api/errors.py | 8 ----- synapse/handlers/e2e_room_keys.py | 41 ++++++++++++----------- synapse/rest/client/v2_alpha/room_keys.py | 2 +- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/synapse/api/errors.py b/synapse/api/errors.py index 0a6e78711f29..48b903374d30 100644 --- a/synapse/api/errors.py +++ b/synapse/api/errors.py @@ -348,14 +348,6 @@ def error_dict(self): ) -def cs_exception(exception): - if isinstance(exception, CodeMessageException): - return exception.error_dict() - else: - logger.error("Unknown exception type: %s", type(exception)) - return {} - - def cs_error(msg, code=Codes.UNKNOWN, **kwargs): """ Utility method for constructing an error response for client-server interactions. diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index bf2a83cc31c9..4e3141dac898 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2017 New Vector Ltd +# Copyright 2017, 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -49,6 +49,11 @@ def get_room_keys(self, user_id, version, room_id=None, session_id=None): room, or a given session. See EndToEndRoomKeyStore.get_e2e_room_keys for full details. + Args: + user_id(str): the user whose keys we're getting + version(str): the version ID of the backup we're getting keys from + room_id(string): room ID to get keys for, for None to get keys for all rooms + session_id(string): session ID to get keys for, for None to get keys for all sessions Returns: A deferred list of dicts giving the session_data and message metadata for these room keys. @@ -72,6 +77,11 @@ def delete_room_keys(self, user_id, version, room_id=None, session_id=None): room or a given session. See EndToEndRoomKeyStore.delete_e2e_room_keys for full details. + Args: + user_id(str): the user whose backup we're deleting + version(str): the version ID of the backup we're deleting + room_id(string): room ID to delete keys for, for None to delete keys for all rooms + session_id(string): session ID to delete keys for, for None to delete keys for all sessions Returns: A deferred of the deletion transaction """ @@ -118,24 +128,24 @@ def upload_room_keys(self, user_id, version, room_keys): # Check that the version we're trying to upload is the current version try: - version_info = yield self._get_version_info_unlocked(user_id) + version_info = yield self.store.get_e2e_room_keys_version_info(user_id) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) else: - raise e + raise if version_info['version'] != version: # Check that the version we're trying to upload actually exists try: - version_info = yield self._get_version_info_unlocked(user_id, version) + version_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) # if we get this far, the version must exist raise RoomKeysVersionError(current_version=version_info['version']) except StoreError as e: if e.code == 404: raise SynapseError(404, "Version '%s' not found" % (version,)) else: - raise e + raise # go through the room_keys. # XXX: this should/could be done concurrently, given we're in a lock. @@ -168,9 +178,9 @@ def _upload_room_key(self, user_id, version, room_id, session_id, room_key): if e.code == 404: pass else: - raise e + raise - if E2eRoomKeysHandler._should_replace_room_key(current_room_key, room_key): + if self._should_replace_room_key(current_room_key, room_key): yield self.store.set_e2e_room_key( user_id, version, room_id, session_id, room_key ) @@ -195,14 +205,14 @@ def _should_replace_room_key(current_room_key, room_key): # purely for legibility. if room_key['is_verified'] and not current_room_key['is_verified']: - pass + return True elif ( room_key['first_message_index'] < current_room_key['first_message_index'] ): - pass + return True elif room_key['forwarded_count'] < current_room_key['forwarded_count']: - pass + return True else: return False return True @@ -256,18 +266,9 @@ def get_version_info(self, user_id, version=None): """ with (yield self._upload_linearizer.queue(user_id)): - res = yield self._get_version_info_unlocked(user_id, version) + res = yield self.store.get_e2e_room_keys_version_info(user_id, version) defer.returnValue(res) - @defer.inlineCallbacks - def _get_version_info_unlocked(self, user_id, version=None): - """Get the info about a given version of the user's backup - without obtaining the upload_linearizer lock. For params see get_version_info - """ - - results = yield self.store.get_e2e_room_keys_version_info(user_id, version) - defer.returnValue(results) - @defer.inlineCallbacks def delete_version(self, user_id, version=None): """Deletes a given version of the user's e2e_room_keys backup diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 539893a5d64b..4807170ea6cb 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2017 New Vector Ltd +# Copyright 2017, 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 86ef9760a786e7de128120c11d623e3303178505 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 11:35:08 +0100 Subject: [PATCH 41/45] Split /room_keys/version into 2 servlets --- synapse/rest/client/v2_alpha/room_keys.py | 30 ++++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 4807170ea6cb..aee641917972 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -240,9 +240,9 @@ def on_DELETE(self, request, room_id, session_id): defer.returnValue((200, {})) -class RoomKeysVersionServlet(RestServlet): +class RoomKeysNewVersionServlet(RestServlet): PATTERNS = client_v2_patterns( - "/room_keys/version(/(?P[^/]+))?$" + "/room_keys/version$" ) def __init__(self, hs): @@ -250,12 +250,12 @@ def __init__(self, hs): Args: hs (synapse.server.HomeServer): server """ - super(RoomKeysVersionServlet, self).__init__() + super(RoomKeysNewVersionServlet, self).__init__() self.auth = hs.get_auth() self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() @defer.inlineCallbacks - def on_POST(self, request, version): + def on_POST(self, request): """ Create a new backup version for this user's room_keys with the given info. The version is allocated by the server and returned to the user @@ -285,10 +285,6 @@ def on_POST(self, request, version): "version": 12345 } """ - - if version: - raise SynapseError(405, "Cannot POST to a specific version") - requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() info = parse_json_object_from_request(request) @@ -301,6 +297,20 @@ def on_POST(self, request, version): # we deliberately don't have a PUT /version, as these things really should # be immutable to avoid people footgunning +class RoomKeysVersionServlet(RestServlet): + PATTERNS = client_v2_patterns( + "/room_keys/version(/(?P[^/]+))?$" + ) + + def __init__(self, hs): + """ + Args: + hs (synapse.server.HomeServer): server + """ + super(RoomKeysVersionServlet, self).__init__() + self.auth = hs.get_auth() + self.e2e_room_keys_handler = hs.get_e2e_room_keys_handler() + @defer.inlineCallbacks def on_GET(self, request, version): """ @@ -320,7 +330,6 @@ def on_GET(self, request, version): "auth_data": "dGhpcyBzaG91bGQgYWN0dWFsbHkgYmUgZW5jcnlwdGVkIGpzb24K" } """ - requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() @@ -346,6 +355,8 @@ def on_DELETE(self, request, version): HTTP/1.1 200 OK {} """ + if version is None: + raise SynapseError(400, "No version specified to delete") requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() @@ -359,3 +370,4 @@ def on_DELETE(self, request, version): def register_servlets(hs, http_server): RoomKeysServlet(hs).register(http_server) RoomKeysVersionServlet(hs).register(http_server) + RoomKeysNewVersionServlet(hs).register(http_server) From bddfad253a2ca8ea0645d10e89055c4ce2ff1c38 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 11:48:02 +0100 Subject: [PATCH 42/45] Don't mangle exceptions --- synapse/rest/client/v2_alpha/room_keys.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index aee641917972..8d006d819da7 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -339,9 +339,7 @@ def on_GET(self, request, version): ) except SynapseError as e: if e.code == 404: - e.errcode = Codes.NOT_FOUND - e.msg = "No backup found" - raise e + raise SynapseError(404, "No backup found", Codes.NOT_FOUND) defer.returnValue((200, info)) @defer.inlineCallbacks @@ -356,7 +354,7 @@ def on_DELETE(self, request, version): {} """ if version is None: - raise SynapseError(400, "No version specified to delete") + raise SynapseError(400, "No version specified to delete", Codes.NOT_FOUND) requester = yield self.auth.get_user_by_req(request, allow_guest=False) user_id = requester.user.to_string() From 306361b31bc0fb229de8db29209b9da38374d6f2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 11:48:56 +0100 Subject: [PATCH 43/45] Misc PR feedback bits --- synapse/storage/e2e_room_keys.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 4d439bb16473..f25ded229520 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import simplejson as json +import json from twisted.internet import defer @@ -76,7 +76,7 @@ def set_e2e_room_key(self, user_id, version, room_id, session_id, room_key): session_id(str): the session whose room_key we're setting room_key(dict): the room_key being set Raises: - StoreError if stuff goes wrong, probably + StoreError """ yield self._simple_upsert( From 8c0ff0287af19cbae40fe7dd413f1e50578908c3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 13:47:43 +0100 Subject: [PATCH 44/45] Linting soothes the savage PEP8 monster --- synapse/handlers/e2e_room_keys.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 4e3141dac898..5edb3cfe0445 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -53,7 +53,8 @@ def get_room_keys(self, user_id, version, room_id=None, session_id=None): user_id(str): the user whose keys we're getting version(str): the version ID of the backup we're getting keys from room_id(string): room ID to get keys for, for None to get keys for all rooms - session_id(string): session ID to get keys for, for None to get keys for all sessions + session_id(string): session ID to get keys for, for None to get keys for all + sessions Returns: A deferred list of dicts giving the session_data and message metadata for these room keys. @@ -80,8 +81,10 @@ def delete_room_keys(self, user_id, version, room_id=None, session_id=None): Args: user_id(str): the user whose backup we're deleting version(str): the version ID of the backup we're deleting - room_id(string): room ID to delete keys for, for None to delete keys for all rooms - session_id(string): session ID to delete keys for, for None to delete keys for all sessions + room_id(string): room ID to delete keys for, for None to delete keys for all + rooms + session_id(string): session ID to delete keys for, for None to delete keys + for all sessions Returns: A deferred of the deletion transaction """ @@ -138,7 +141,9 @@ def upload_room_keys(self, user_id, version, room_keys): if version_info['version'] != version: # Check that the version we're trying to upload actually exists try: - version_info = yield self.store.get_e2e_room_keys_version_info(user_id, version) + version_info = yield self.store.get_e2e_room_keys_version_info( + user_id, version, + ) # if we get this far, the version must exist raise RoomKeysVersionError(current_version=version_info['version']) except StoreError as e: From a45f2c3a002dd50ba4dc6dcbe45a51f2da4057b2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 12 Oct 2018 14:33:55 +0100 Subject: [PATCH 45/45] missed one --- synapse/rest/client/v2_alpha/room_keys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 8d006d819da7..45b5817d8b7e 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -297,6 +297,7 @@ def on_POST(self, request): # we deliberately don't have a PUT /version, as these things really should # be immutable to avoid people footgunning + class RoomKeysVersionServlet(RestServlet): PATTERNS = client_v2_patterns( "/room_keys/version(/(?P[^/]+))?$"