Fail prediction on unhandled error (#1981)

If the runner fails to handle an error this gets bubbled up to the
worker, which logs a warning and then merrily carries on. Errors
bubbling up in this way include things like failing to upload a file,
which should fail the prediction. Currently if a model fails to upload
one or more files for the prediction, the prediction will still be
marked as succeeded, but it won't have any files in the output.

python/cog/server/runner.py

@@ -372,21 +372,24 @@ def canceled(self) -> None:
         self._send_webhook(schema.WebhookEvent.COMPLETED)
 
     def handle_event(self, event: _PublicEventType) -> None:
-        if isinstance(event, Log):
-            self.append_logs(event.message)
-        elif isinstance(event, PredictionOutputType):
-            self.set_output_type(multi=event.multi)
-        elif isinstance(event, PredictionOutput):
-            self.append_output(event.payload)
-        elif isinstance(event, Done):  # pyright: ignore reportUnnecessaryIsinstance
-            if event.canceled:
-                self.canceled()
-            elif event.error:
-                self.failed(error=str(event.error_detail))
-            else:
-                self.succeeded()
-        else:  # shouldn't happen, exhausted the type
-            self._log.warn("received unexpected event during predict", data=event)
+        try:
+            if isinstance(event, Log):
+                self.append_logs(event.message)
+            elif isinstance(event, PredictionOutputType):
+                self.set_output_type(multi=event.multi)
+            elif isinstance(event, PredictionOutput):
+                self.append_output(event.payload)
+            elif isinstance(event, Done):  # pyright: ignore reportUnnecessaryIsinstance
+                if event.canceled:
+                    self.canceled()
+                elif event.error:
+                    self.failed(error=str(event.error_detail))
+                else:
+                    self.succeeded()
+            else:  # shouldn't happen, exhausted the type
+                self._log.warn("received unexpected event during predict", data=event)
+        except Exception as e:
+            self.failed(str(e))
 
     def _set_completed_at(self) -> None:
         self._p.completed_at = datetime.now(tz=timezone.utc)