GPUDevice: Add recovery from lost device

src/core/gpu.cpp

@@ -1993,7 +1993,7 @@ void GPU::SetDisplayTexture(GPUTexture* texture, GPUTexture* depth_buffer, s32 v
   m_display_texture_view_height = view_height;
 }
 
-bool GPU::PresentDisplay()
+GPUDevice::PresentResult GPU::PresentDisplay()
 {
   FlushRender();
 
@@ -2004,7 +2004,8 @@ bool GPU::PresentDisplay()
   return RenderDisplay(nullptr, display_rect, draw_rect, !g_settings.debugging.show_vram);
 }
 
-bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const GSVector4i draw_rect, bool postfx)
+GPUDevice::PresentResult GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect,
+                                            const GSVector4i draw_rect, bool postfx)
 {
   GL_SCOPE_FMT("RenderDisplay: {}", draw_rect);
 
@@ -2027,10 +2028,15 @@ bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const
 
     // Now we can apply the post chain.
     GPUTexture* post_output_texture = PostProcessing::InternalChain.GetOutputTexture();
-    if (PostProcessing::InternalChain.Apply(display_texture, m_display_depth_buffer, post_output_texture,
-                                            GSVector4i(0, 0, display_texture_view_width, display_texture_view_height),
-                                            display_texture_view_width, display_texture_view_height,
-                                            m_crtc_state.display_width, m_crtc_state.display_height))
+    if (const GPUDevice::PresentResult pres = PostProcessing::InternalChain.Apply(
+          display_texture, m_display_depth_buffer, post_output_texture,
+          GSVector4i(0, 0, display_texture_view_width, display_texture_view_height), display_texture_view_width,
+          display_texture_view_height, m_crtc_state.display_width, m_crtc_state.display_height);
+        pres != GPUDevice::PresentResult::OK)
+    {
+      return pres;
+    }
+    else
     {
       display_texture_view_x = 0;
       display_texture_view_y = 0;
@@ -2057,8 +2063,9 @@ bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const
   {
     if (target)
       g_gpu_device->SetRenderTarget(target);
-    else if (!g_gpu_device->BeginPresent(false))
-      return false;
+    else if (const GPUDevice::PresentResult pres = g_gpu_device->BeginPresent(false);
+             pres != GPUDevice::PresentResult::OK)
+      return pres;
   }
 
   if (display_texture)
@@ -2167,7 +2174,9 @@ bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const
                                               m_crtc_state.display_height);
   }
   else
-    return true;
+  {
+    return GPUDevice::PresentResult::OK;
+  }
 }
 
 bool GPU::SendDisplayToMediaCapture(MediaCapture* cap)
@@ -2186,7 +2195,7 @@ bool GPU::SendDisplayToMediaCapture(MediaCapture* cap)
   // Not cleared by RenderDisplay().
   g_gpu_device->ClearRenderTarget(target, GPUDevice::DEFAULT_CLEAR_COLOR);
 
-  if (!RenderDisplay(target, display_rect, draw_rect, postfx)) [[unlikely]]
+  if (RenderDisplay(target, display_rect, draw_rect, postfx) != GPUDevice::PresentResult::OK) [[unlikely]]
     return false;
 
   return cap->DeliverVideoFrame(target);

src/core/gpu.h

@@ -8,6 +8,7 @@
 #include "timing_event.h"
 #include "types.h"
 
+#include "util/gpu_device.h"
 #include "util/gpu_texture.h"
 
 #include "common/bitfield.h"
@@ -233,7 +234,7 @@ class GPU
                               bool show_osd_message);
 
   /// Draws the current display texture, with any post-processing.
-  bool PresentDisplay();
+  GPUDevice::PresentResult PresentDisplay();
 
   /// Sends the current frame to media capture.
   bool SendDisplayToMediaCapture(MediaCapture* cap);
@@ -630,7 +631,8 @@ class GPU
   void SetDisplayTexture(GPUTexture* texture, GPUTexture* depth_texture, s32 view_x, s32 view_y, s32 view_width,
                          s32 view_height);
 
-  bool RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const GSVector4i draw_rect, bool postfx);
+  GPUDevice::PresentResult RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const GSVector4i draw_rect,
+                                         bool postfx);
 
   bool Deinterlace(u32 field, u32 line_skip);
   bool DeinterlaceExtractField(u32 dst_bufidx, GPUTexture* src, u32 x, u32 y, u32 width, u32 height, u32 line_skip);

src/core/imgui_overlays.cpp

@@ -160,7 +160,7 @@ void Host::DisplayLoadingScreen(const char* message, int progress_min /*= -1*/,
 
   // TODO: Glass effect or something.
 
-  if (g_gpu_device->BeginPresent(false))
+  if (g_gpu_device->BeginPresent(false) == GPUDevice::PresentResult::OK)
   {
     g_gpu_device->RenderImGui();
     g_gpu_device->EndPresent(false);

src/core/system.cpp

@@ -162,6 +162,7 @@ static void DestroySystem();
 
 static bool CreateGPU(GPURenderer renderer, bool is_switching, Error* error);
 static bool RecreateGPU(GPURenderer renderer, bool force_recreate_device = false, bool update_display = true);
+static void HandleHostGPUDeviceLost();
 
 /// Updates the throttle period, call when target emulation speed changes.
 static void UpdateThrottlePeriod();
@@ -1202,6 +1203,45 @@ bool System::RecreateGPU(GPURenderer renderer, bool force_recreate_device, bool
   return true;
 }
 
+void System::HandleHostGPUDeviceLost()
+{
+  static Common::Timer::Value s_last_gpu_reset_time = 0;
+  static constexpr float MIN_TIME_BETWEEN_RESETS = 15.0f;
+
+  // If we're constantly crashing on something in particular, we don't want to end up in an
+  // endless reset loop.. that'd probably end up leaking memory and/or crashing us for other
+  // reasons. So just abort in such case.
+  const Common::Timer::Value current_time = Common::Timer::GetCurrentValue();
+  if (s_last_gpu_reset_time != 0 &&
+      Common::Timer::ConvertValueToSeconds(current_time - s_last_gpu_reset_time) < MIN_TIME_BETWEEN_RESETS)
+  {
+    Panic("Host GPU lost too many times, device is probably completely wedged.");
+  }
+  s_last_gpu_reset_time = current_time;
+
+  // Little bit janky, but because the device is lost, the VRAM readback is going to give us garbage.
+  // So back up what we have, it's probably missing bits, but whatever...
+  DynamicHeapArray<u8> vram_backup(VRAM_SIZE);
+  std::memcpy(vram_backup.data(), g_vram, VRAM_SIZE);
+
+  // Device lost, something went really bad.
+  // Let's just toss out everything, and try to hobble on.
+  if (!RecreateGPU(g_gpu->IsHardwareRenderer() ? g_settings.gpu_renderer : GPURenderer::Software, true, false))
+  {
+    Panic("Failed to recreate GS device after loss.");
+    return;
+  }
+
+  // Restore backed-up VRAM.
+  std::memcpy(g_vram, vram_backup.data(), VRAM_SIZE);
+
+  // First frame after reopening is definitely going to be trash, so skip it.
+  Host::AddIconOSDMessage(
+    "HostGPUDeviceLost", ICON_EMOJI_WARNING,
+    TRANSLATE_STR("System", "Host GPU device encountered an error and has recovered. This may cause broken rendering."),
+    Host::OSD_CRITICAL_ERROR_DURATION);
+}
+
 void System::LoadSettings(bool display_osd_messages)
 {
   std::unique_lock<std::mutex> lock = Host::GetSettingsLock();
@@ -5710,13 +5750,13 @@ bool System::PresentDisplay(bool skip_present, bool explicit_present)
   ImGuiManager::RenderOverlayWindows();
   ImGuiManager::RenderDebugWindows();
 
-  bool do_present;
+  GPUDevice::PresentResult pres;
   if (g_gpu && !skip_present)
-    do_present = g_gpu->PresentDisplay();
+    pres = g_gpu->PresentDisplay();
   else
-    do_present = g_gpu_device->BeginPresent(skip_present);
+    pres = g_gpu_device->BeginPresent(skip_present);
 
-  if (do_present)
+  if (pres == GPUDevice::PresentResult::OK)
   {
     g_gpu_device->RenderImGui();
     g_gpu_device->EndPresent(explicit_present);
@@ -5729,13 +5769,16 @@ bool System::PresentDisplay(bool skip_present, bool explicit_present)
   }
   else
   {
+    if (pres == GPUDevice::PresentResult::DeviceLost) [[unlikely]]
+      HandleHostGPUDeviceLost();
+
     // Still need to kick ImGui or it gets cranky.
     ImGui::Render();
   }
 
   ImGuiManager::NewFrame();
 
-  return do_present;
+  return (pres == GPUDevice::PresentResult::OK);
 }
 
 void System::InvalidateDisplay()

src/util/d3d11_device.cpp

@@ -639,17 +639,17 @@ void D3D11Device::SetVSyncMode(GPUVSyncMode mode, bool allow_present_throttle)
   }
 }
 
-bool D3D11Device::BeginPresent(bool skip_present, u32 clear_color)
+GPUDevice::PresentResult D3D11Device::BeginPresent(bool skip_present, u32 clear_color)
 {
   if (skip_present)
-    return false;
+    return PresentResult::SkipPresent;
 
   if (!m_swap_chain)
   {
     // Note: Really slow on Intel...
     m_context->Flush();
     TrimTexturePool();
-    return false;
+    return PresentResult::SkipPresent;
   }
 
   // Check if we lost exclusive fullscreen. If so, notify the host, so it can switch to windowed mode.
@@ -660,7 +660,7 @@ bool D3D11Device::BeginPresent(bool skip_present, u32 clear_color)
   {
     Host::SetFullscreen(false);
     TrimTexturePool();
-    return false;
+    return PresentResult::SkipPresent;
   }
 
   // When using vsync, the time here seems to include the time for the buffer to become available.
@@ -677,7 +677,7 @@ bool D3D11Device::BeginPresent(bool skip_present, u32 clear_color)
   m_current_render_pass_flags = GPUPipeline::NoRenderPassFlags;
   std::memset(m_current_render_targets.data(), 0, sizeof(m_current_render_targets));
   m_current_depth_target = nullptr;
-  return true;
+  return PresentResult::OK;
 }
 
 void D3D11Device::EndPresent(bool explicit_present)

src/util/d3d11_device.h

@@ -104,7 +104,7 @@ class D3D11Device final : public GPUDevice
   bool SetGPUTimingEnabled(bool enabled) override;
   float GetAndResetAccumulatedGPUTime() override;
 
-  bool BeginPresent(bool skip_present, u32 clear_color) override;
+  PresentResult BeginPresent(bool skip_present, u32 clear_color) override;
   void EndPresent(bool explicit_present) override;
   void SubmitPresent() override;
 

src/util/d3d12_device.cpp

@@ -532,6 +532,9 @@ ID3D12GraphicsCommandList4* D3D12Device::GetInitCommandList()
 
 void D3D12Device::SubmitCommandList(bool wait_for_completion)
 {
+  if (m_device_was_lost) [[unlikely]]
+    return;
+
   CommandList& res = m_command_lists[m_current_command_list];
   HRESULT hr;
 
@@ -553,7 +556,8 @@ void D3D12Device::SubmitCommandList(bool wait_for_completion)
     if (FAILED(hr)) [[unlikely]]
     {
       ERROR_LOG("Closing init command list failed with HRESULT {:08X}", static_cast<unsigned>(hr));
-      Panic("TODO cannot continue");
+      m_device_was_lost = true;
+      return;
     }
   }
 
@@ -562,7 +566,8 @@ void D3D12Device::SubmitCommandList(bool wait_for_completion)
   if (FAILED(hr)) [[unlikely]]
   {
     ERROR_LOG("Closing main command list failed with HRESULT {:08X}", static_cast<unsigned>(hr));
-    Panic("TODO cannot continue");
+    m_device_was_lost = true;
+    return;
   }
 
   if (res.init_list_used)
@@ -578,7 +583,12 @@ void D3D12Device::SubmitCommandList(bool wait_for_completion)
 
   // Update fence when GPU has completed.
   hr = m_command_queue->Signal(m_fence.Get(), res.fence_counter);
-  DebugAssertMsg(SUCCEEDED(hr), "Signal fence");
+  if (FAILED(hr))
+  {
+    ERROR_LOG("Signal command queue fence failed with HRESULT {:08X}", static_cast<unsigned>(hr));
+    m_device_was_lost = true;
+    return;
+  }
 
   MoveToNextCommandList();
 
@@ -606,6 +616,9 @@ void D3D12Device::SubmitCommandListAndRestartRenderPass(const std::string_view r
 
 void D3D12Device::WaitForFence(u64 fence)
 {
+  if (m_device_was_lost) [[unlikely]]
+    return;
+
   if (m_completed_fence_value >= fence)
     return;
 
@@ -1110,20 +1123,23 @@ void D3D12Device::SetVSyncMode(GPUVSyncMode mode, bool allow_present_throttle)
   }
 }
 
-bool D3D12Device::BeginPresent(bool frame_skip, u32 clear_color)
+GPUDevice::PresentResult D3D12Device::BeginPresent(bool frame_skip, u32 clear_color)
 {
   if (InRenderPass())
     EndRenderPass();
 
+  if (m_device_was_lost) [[unlikely]]
+    return PresentResult::DeviceLost;
+
   if (frame_skip)
-    return false;
+    return PresentResult::SkipPresent;
 
   // If we're running surfaceless, kick the command buffer so we don't run out of descriptors.
   if (!m_swap_chain)
   {
     SubmitCommandList(false);
     TrimTexturePool();
-    return false;
+    return PresentResult::SkipPresent;
   }
 
   // TODO: Check if the device was lost.
@@ -1136,11 +1152,11 @@ bool D3D12Device::BeginPresent(bool frame_skip, u32 clear_color)
   {
     Host::RunOnCPUThread([]() { Host::SetFullscreen(false); });
     TrimTexturePool();
-    return false;
+    return PresentResult::SkipPresent;
   }
 
   BeginSwapChainRenderPass(clear_color);
-  return true;
+  return PresentResult::OK;
 }
 
 void D3D12Device::EndPresent(bool explicit_present)
@@ -1165,6 +1181,8 @@ void D3D12Device::EndPresent(bool explicit_present)
 void D3D12Device::SubmitPresent()
 {
   DebugAssert(m_swap_chain);
+  if (m_device_was_lost) [[unlikely]]
+    return;
 
   const UINT sync_interval = static_cast<UINT>(m_vsync_mode == GPUVSyncMode::FIFO);
   const UINT flags = (m_vsync_mode == GPUVSyncMode::Disabled && m_using_allow_tearing) ? DXGI_PRESENT_ALLOW_TEARING : 0;

src/util/d3d12_device.h

@@ -126,7 +126,7 @@ class D3D12Device final : public GPUDevice
   bool SetGPUTimingEnabled(bool enabled) override;
   float GetAndResetAccumulatedGPUTime() override;
 
-  bool BeginPresent(bool skip_present, u32 clear_color) override;
+  PresentResult BeginPresent(bool skip_present, u32 clear_color) override;
   void EndPresent(bool explicit_present) override;
   void SubmitPresent() override;
 
@@ -300,6 +300,7 @@ class D3D12Device final : public GPUDevice
   bool m_allow_tearing_supported = false;
   bool m_using_allow_tearing = false;
   bool m_is_exclusive_fullscreen = false;
+  bool m_device_was_lost = false;
 
   D3D12DescriptorHeapManager m_descriptor_heap_manager;
   D3D12DescriptorHeapManager m_rtv_heap_manager;

src/util/gpu_device.h

@@ -481,6 +481,13 @@ class GPUDevice
     Full
   };
 
+  enum class PresentResult : u32
+  {
+    OK,
+    SkipPresent,
+    DeviceLost,
+  };
+
   struct Features
   {
     bool dual_source_blend : 1;
@@ -702,7 +709,7 @@ class GPUDevice
   virtual void DrawIndexedWithBarrier(u32 index_count, u32 base_index, u32 base_vertex, DrawBarrier type) = 0;
 
   /// Returns false if the window was completely occluded.
-  virtual bool BeginPresent(bool skip_present, u32 clear_color = DEFAULT_CLEAR_COLOR) = 0;
+  virtual PresentResult BeginPresent(bool skip_present, u32 clear_color = DEFAULT_CLEAR_COLOR) = 0;
   virtual void EndPresent(bool explicit_submit) = 0;
   virtual void SubmitPresent() = 0;
 

src/util/metal_device.h

@@ -265,7 +265,7 @@ class MetalDevice final : public GPUDevice
 
   void SetVSyncMode(GPUVSyncMode mode, bool allow_present_throttle) override;
 
-  bool BeginPresent(bool skip_present, u32 clear_color) override;
+  PresentResult BeginPresent(bool skip_present, u32 clear_color) override;
   void EndPresent(bool explicit_submit) override;
   void SubmitPresent() override;