diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py
index 3fac256881d0..9a3706844d81 100644
--- a/synapse/handlers/federation_event.py
+++ b/synapse/handlers/federation_event.py
@@ -866,11 +866,6 @@ async def _process_pulled_event(
                 event.room_id, event_id, str(err)
             )
             return
-        except Exception as exc:
-            await self._store.record_event_failed_pull_attempt(
-                event.room_id, event_id, str(exc)
-            )
-            raise exc
 
         try:
             try:
@@ -913,7 +908,7 @@ async def _process_pulled_event(
                 logger.warning("Pulled event %s failed history check.", event_id)
             else:
                 raise
-        except Exception as exc:
+        except (FederationError, RuntimeError, InvalidResponseError) as exc:
             await self._store.record_event_failed_pull_attempt(
                 event.room_id, event_id, str(exc)
             )