Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Fix a number of logged errors caused by remote servers being down. #10400

Merged
merged 7 commits into from
Jul 15, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/10400.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix a number of logged errors caused by remote servers being down.
5 changes: 4 additions & 1 deletion synapse/handlers/directory.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
CodeMessageException,
Codes,
NotFoundError,
RequestSendFailed,
ShadowBanError,
StoreError,
SynapseError,
Expand Down Expand Up @@ -252,12 +253,14 @@ async def get_association(self, room_alias: RoomAlias) -> JsonDict:
retry_on_dns_fail=False,
ignore_backoff=True,
)
except RequestSendFailed:
raise SynapseError(502, "Failed to fetch alias")
except CodeMessageException as e:
logging.warning("Error retrieving alias")
if e.code == 404:
fed_result = None
else:
raise
raise SynapseError(502, "Failed to fetch alias")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we log the original error as well? (or maybe it's already logged as part of the HTTP client logs, I can't remember)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good Q. We should log response success/errors in the http clients, so I don't think its necessary here.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW, re-raising here implicitly adds from e, so the traceback of the original exception will be available to anyone who catches it.


if fed_result and "room_id" in fed_result and "servers" in fed_result:
room_id = fed_result["room_id"]
Expand Down
25 changes: 16 additions & 9 deletions synapse/handlers/federation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1414,12 +1414,15 @@ async def send_invite(self, target_host: str, event: EventBase) -> EventBase:

Invites must be signed by the invitee's server before distribution.
"""
pdu = await self.federation_client.send_invite(
destination=target_host,
room_id=event.room_id,
event_id=event.event_id,
pdu=event,
)
try:
pdu = await self.federation_client.send_invite(
destination=target_host,
room_id=event.room_id,
event_id=event.event_id,
pdu=event,
)
except RequestSendFailed:
raise SynapseError(502, f"Can't connect to server {target_host}")

return pdu

Expand Down Expand Up @@ -3031,9 +3034,13 @@ async def exchange_third_party_invite(
await member_handler.send_membership_event(None, event, context)
else:
destinations = {x.split(":", 1)[-1] for x in (sender_user_id, room_id)}
await self.federation_client.forward_third_party_invite(
destinations, room_id, event_dict
)

try:
await self.federation_client.forward_third_party_invite(
destinations, room_id, event_dict
)
except (RequestSendFailed, HttpResponseException):
raise SynapseError(502, "Failed to forward third party invite")

async def on_exchange_third_party_invite_request(
self, event_dict: JsonDict
Expand Down
26 changes: 17 additions & 9 deletions synapse/handlers/room_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,12 @@
from unpaddedbase64 import decode_base64, encode_base64

from synapse.api.constants import EventTypes, HistoryVisibility, JoinRules
from synapse.api.errors import Codes, HttpResponseException
from synapse.api.errors import (
Codes,
HttpResponseException,
RequestSendFailed,
SynapseError,
)
from synapse.types import JsonDict, ThirdPartyInstanceID
from synapse.util.caches.descriptors import cached
from synapse.util.caches.response_cache import ResponseCache
Expand Down Expand Up @@ -417,14 +422,17 @@ async def _get_remote_list_cached(
repl_layer = self.hs.get_federation_client()
if search_filter:
# We can't cache when asking for search
return await repl_layer.get_public_rooms(
server_name,
limit=limit,
since_token=since_token,
search_filter=search_filter,
include_all_networks=include_all_networks,
third_party_instance_id=third_party_instance_id,
)
try:
return await repl_layer.get_public_rooms(
server_name,
limit=limit,
since_token=since_token,
search_filter=search_filter,
include_all_networks=include_all_networks,
third_party_instance_id=third_party_instance_id,
)
except (RequestSendFailed, HttpResponseException):
raise SynapseError(502, "Failed to fetch room list")

key = (
server_name,
Expand Down
28 changes: 28 additions & 0 deletions synapse/http/matrixfederationclient.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
from twisted.internet.error import DNSLookupError
from twisted.internet.interfaces import IReactorTime
from twisted.internet.task import _EPSILON, Cooperator
from twisted.web.client import ResponseFailed
from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer, IResponse

Expand Down Expand Up @@ -262,6 +263,15 @@ async def _handle_response(
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except ResponseFailed as e:
logger.warning(
"{%s} [%s] Failed to read response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except Exception as e:
logger.warning(
"{%s} [%s] Error reading response %s %s: %s",
Expand Down Expand Up @@ -1137,6 +1147,24 @@ async def get_file(
msg,
)
raise SynapseError(502, msg, Codes.TOO_LARGE)
except defer.TimeoutError as e:
logger.warning(
"{%s} [%s] Timed out reading response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except ResponseFailed as e:
logger.warning(
"{%s} [%s] Failed to read response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except Exception as e:
logger.warning(
"{%s} [%s] Error reading response: %s",
Expand Down