Potential macOS Metal & vsync improvements

Source/Core/VideoBackends/Vulkan/Renderer.cpp

@@ -1711,10 +1711,20 @@ void Renderer::SetViewport()
 	StateTracker::GetInstance()->SetViewport(viewport);
 }
 
+void Renderer::CacheSurfaceHandle(void* new_surface_handle)
+{
+	m_cached_surface_handle = new_surface_handle;
+}
+
 void Renderer::ChangeSurface(void* new_surface_handle)
 {
 	// Called by the main thread when the window is resized.
+#if defined(__APPLE__)
+	m_new_surface_handle = m_cached_surface_handle;
+#else
 	m_new_surface_handle = new_surface_handle;
+#endif
+
 	m_surface_needs_change.Set();
 	m_surface_changed.Set();
 }

Source/Core/VideoBackends/Vulkan/Renderer.h

@@ -67,6 +67,7 @@ class Renderer : public ::Renderer
 	void SetViewport() override;
 
 	void ChangeSurface(void* new_surface_handle) override;
+	void CacheSurfaceHandle(void* new_surface_handle) override;
 
 private:
 	bool CreateSemaphores();

Source/Core/VideoBackends/Vulkan/SwapChain.cpp

@@ -369,6 +369,7 @@ bool SwapChain::CreateSwapChain()
 
 	res =
 		vkCreateSwapchainKHR(g_vulkan_context->GetDevice(), &swap_chain_info, nullptr, &m_swap_chain);
+
 	if (res != VK_SUCCESS)
 	{
 		LOG_VULKAN_ERROR(res, "vkCreateSwapchainKHR failed: ");

Source/Core/VideoBackends/Vulkan/main.cpp

@@ -27,10 +27,15 @@
 
 #if defined(VK_USE_PLATFORM_METAL_EXT)
 #include <objc/message.h>
+#include <CoreGraphics/CGBase.h>
+#include <CoreGraphics/CGGeometry.h>
 #endif
 
 namespace Vulkan
 {
+
+static void* s_metal_view_handle = nullptr;
+
 void VideoBackend::InitBackendInfo()
 {
 	VulkanContext::PopulateBackendInfo(&g_Config);
@@ -117,8 +122,16 @@ bool VideoBackend::Initialize(void* window_handle)
 		enable_validation_layer = false;
 	}
 
+	// On macOS, we want to get the subview that hosts the rendering layer. Other platforms
+	// render through to the underlying view with no issues.
+#if defined(VK_USE_PLATFORM_METAL_EXT)
+	void* win_handle = s_metal_view_handle; 
+#else
+	void* win_handle = window_handle;
+#endif
+
 	// Create Vulkan instance, needed before we can create a surface.
-	bool enable_surface = window_handle != nullptr;
+	bool enable_surface = win_handle != nullptr;
 	bool enable_debug_reports = ShouldEnableDebugReports(enable_validation_layer);
 	VkInstance instance = VulkanContext::CreateVulkanInstance(enable_surface, enable_debug_reports,
 		enable_validation_layer);
@@ -144,7 +157,7 @@ bool VideoBackend::Initialize(void* window_handle)
 	VkSurfaceKHR surface = VK_NULL_HANDLE;
 	if (enable_surface)
 	{
-		surface = SwapChain::CreateVulkanSurface(instance, window_handle);
+		surface = SwapChain::CreateVulkanSurface(instance, win_handle);
 		if (surface == VK_NULL_HANDLE)
 		{
 			PanicAlert("Failed to create Vulkan surface.");
@@ -192,7 +205,7 @@ bool VideoBackend::Initialize(void* window_handle)
 	std::unique_ptr<SwapChain> swap_chain;
 	if (surface != VK_NULL_HANDLE)
 	{
-		swap_chain = SwapChain::Create(window_handle, surface, g_Config.IsVSync());
+		swap_chain = SwapChain::Create(win_handle, surface, g_Config.IsVSync());
 		if (!swap_chain)
 		{
 			PanicAlert("Failed to create Vulkan swap chain.");
@@ -217,6 +230,13 @@ bool VideoBackend::Initialize(void* window_handle)
 	g_framebuffer_manager = std::make_unique<FramebufferManager>();
 	g_renderer = std::make_unique<Renderer>(std::move(swap_chain));
 	g_renderer->Init();
+
+	// We cache this on the renderer if it's Metal, as fullscreen changes need to use the
+	// correct rendering layer to handle swap chain recreation.
+#if defined(VK_USE_PLATFORM_METAL_EXT)
+	g_renderer->CacheSurfaceHandle(s_metal_view_handle);
+#endif
+
 	// Invoke init methods on main wrapper classes.
 	// These have to be done before the others because the destructors
 	// for the remaining classes may call methods on these.
@@ -282,6 +302,10 @@ void VideoBackend::Shutdown()
 	UnloadVulkanLibrary();
 
 	ShutdownShared();
+
+#if defined(VK_USE_PLATFORM_METAL_EXT)
+	s_metal_view_handle = nullptr;
+#endif
 }
 
 void VideoBackend::Video_Cleanup()
@@ -301,45 +325,140 @@ void VideoBackend::Video_Cleanup()
 	CleanupShared();
 }
 
-void VideoBackend::PrepareWindow(void* window_handle) {
 #if defined(VK_USE_PLATFORM_METAL_EXT)
-	id view = reinterpret_cast<id>(window_handle);
+// This is injected as a subclass method on the custom layer view, and
+// tells macOS to avoid `drawRect:` and opt for direct layer updating instead.
+BOOL wantsUpdateLayer(id self, SEL _cmd, id sender)
+{
+	return YES;
+}
 
-	// This is kinda messy, but it avoids having to write Objective C++ just to create a metal layer.
-	//id view = reinterpret_cast<id>(wsi.render_surface);
-	Class clsCAMetalLayer = objc_getClass("CAMetalLayer");
-	if (!clsCAMetalLayer)
-	{
-	ERROR_LOG(VIDEO, "Failed to get CAMetalLayer class.");
-	return;
-	}
+// Used by some internals, but ideally never gets called to begin with.
+Class getLayerClass(id self, SEL _cmd)
+{
+	Class clsCAMetalLayerClass = objc_getClass("CAMetalLayer");
+	return clsCAMetalLayerClass;
+}
+
+// When `wantsLayer` is true, this method is invoked to create the actual backing layer.
+id makeBackingLayer(id self, SEL _cmd)
+{
+	Class metalLayerClass = objc_getClass("CAMetalLayer");
 
-	// [CAMetalLayer layer]
-	id layer = reinterpret_cast<id (*)(Class, SEL)>(objc_msgSend)(objc_getClass("CAMetalLayer"),
-	                                                            sel_getUid("layer"));
-	if (!layer)
+	// This should only be possible prior to macOS 10.14, but worth logging regardless.
+	if (!metalLayerClass)
 	{
-	ERROR_LOG(VIDEO, "Failed to create Metal layer.");
-	return;
+		ERROR_LOG(VIDEO, "Failed to get CAMetalLayer class.");
 	}
 
-	// [view setWantsLayer:YES]
-	reinterpret_cast<void (*)(id, SEL, BOOL)>(objc_msgSend)(view, sel_getUid("setWantsLayer:"), YES);
-
-	// [view setLayer:layer]
-	reinterpret_cast<void (*)(id, SEL, id)>(objc_msgSend)(view, sel_getUid("setLayer:"), layer);
+	id layer = reinterpret_cast<id (*)(Class, SEL)>(objc_msgSend)(metalLayerClass, sel_getUid("layer"));
 
-	// NSScreen* screen = [NSScreen mainScreen]
 	id screen = reinterpret_cast<id (*)(Class, SEL)>(objc_msgSend)(objc_getClass("NSScreen"),
 	                                                             sel_getUid("mainScreen"));
 
 	// CGFloat factor = [screen backingScaleFactor]
-	double factor =
-	  reinterpret_cast<double (*)(id, SEL)>(objc_msgSend)(screen, sel_getUid("backingScaleFactor"));
+	double factor = reinterpret_cast<double (*)(id, SEL)>(objc_msgSend)(screen, sel_getUid("backingScaleFactor"));
 
 	// layer.contentsScale = factor
 	reinterpret_cast<void (*)(id, SEL, double)>(objc_msgSend)(layer, sel_getUid("setContentsScale:"),
 	                                                        factor);
+
+	// This is an oddity, but alright. The SwapChain is already configured to be respective of Vsync, but the underlying
+	// CAMetalLayer *also* needs to be instructed to respect it. This defaults to YES; if we're not supposed to have vsync
+	// enabled, then we need to flip this.
+	//
+	// Notably, some M1 Macs have issues without this logic.
+	// 
+	// I have absolutely no clue why this works, as MoltenVK also sets this property. Setting it before giving the layer
+	// to MoltenVK seems to make it stick, though.
+	if (!g_Config.IsVSync())
+	{
+		// Explicitly tells the underlying layer to NOT use vsync.
+		// [view setDisplaySyncEnabled:NO]
+		reinterpret_cast<void (*)(id, SEL, BOOL)>(objc_msgSend)(layer, sel_getUid("setDisplaySyncEnabled:"), NO);        
+	}
+
+	// CAMetalLayer is triple-buffered by default; we can lower this to double buffering. 
+	//
+	// (The only acceptable values are `2` or `3`). Typically it's only iMacs that can handle this, so we'll just 
+	// enable an ENV variable for it and document it on the wiki.
+	if (getenv("SLP_METAL_DOUBLE_BUFFER") != NULL)
+	{
+		reinterpret_cast<void (*)(id, SEL, BOOL)>(objc_msgSend)(layer, sel_getUid("setMaximumDrawableCount:"), 2);
+	}
+
+	return layer;
+}
+
+constexpr char kSLPMetalLayerViewClassName[] = "SLPMetalLayerViewClass";
+
+// This method injects a custom NSView subclass into the Objective-C runtime.
+//
+// The reason this is done is due to wanting to bypass NSView's `drawRect:` for Metal rendering
+// purposes. To do this, it's not enough to just set `wantsLayer` to true - we need to also implement
+// a few subclass methods, and tell the system we *want* the fast path.
+//
+// We have to inject a custom subclass as we can't modify the view (window_handle) in `PrepareWindow`,
+// as that's a wxWidgets handle that relies on `drawRect:` being called for things to work. To work
+// around this, we simply take the `window_handle` (i.e the view), create an instance of our `SLPMetalLayerView`,
+// and attach that as a child view. `SLPMetalLayerView` should get the fast path, while everything else should
+// stay golden.
+Class getSLPMetalLayerViewClassType()
+{
+	Class SLPMetalLayerViewClass = objc_getClass(kSLPMetalLayerViewClassName);
+
+	if (SLPMetalLayerViewClass == nullptr)
+	{
+#ifdef IS_PLAYBACK
+		// These are disabled on Playback builds for now, as M1 devices running Playback under Rosetta 2
+		// seem to hit a race condition with asynchronous queue submits. Rendering takes a slight hit but
+		// this matters less in playback, and it's still better than OpenGL.
+		//setenv("MVK_CONFIG_SYNCHRONOUS_QUEUE_SUBMITS", "0", 0);
+		//setenv("MVK_CONFIG_PRESENT_WITH_COMMAND_BUFFER", "0", 0);
+#else
+		// This does a one-time opt-in to a MVK flag that seems to universally help in Ishiiruka.
+		// (mainline should not need this)
+		setenv("MVK_CONFIG_SYNCHRONOUS_QUEUE_SUBMITS", "0", 0);
+		setenv("MVK_CONFIG_PRESENT_WITH_COMMAND_BUFFER", "0", 0);		
+#endif
+		SLPMetalLayerViewClass = objc_allocateClassPair(
+			(Class)objc_getClass("NSView"),
+			kSLPMetalLayerViewClassName,
+			0
+		);
+
+		class_addMethod(SLPMetalLayerViewClass, sel_getUid("layerClass"), (IMP)getLayerClass, "@:@");
+		class_addMethod(SLPMetalLayerViewClass, sel_getUid("wantsUpdateLayer"), (IMP)wantsUpdateLayer, "v@:");
+		class_addMethod(SLPMetalLayerViewClass, sel_getUid("makeBackingLayer"), (IMP)makeBackingLayer, "@:@");
+		class_addMethod(SLPMetalLayerViewClass, sel_getUid("isOpaque"), (IMP)wantsUpdateLayer, "v@:");
+		objc_registerClassPair(SLPMetalLayerViewClass);
+	}
+
+	return SLPMetalLayerViewClass;
+}
+#endif
+
+void VideoBackend::PrepareWindow(void* window_handle) {
+#if defined(VK_USE_PLATFORM_METAL_EXT)
+	id view = reinterpret_cast<id>(window_handle);
+
+	CGRect (*sendRectFn)(id receiver, SEL operation);
+	sendRectFn = (CGRect(*)(id, SEL))objc_msgSend_stret;
+	CGRect frame = sendRectFn(view, sel_getUid("frame"));
+
+	Class SLPMetalLayerViewClass = getSLPMetalLayerViewClassType();
+	id alloc = reinterpret_cast<id (*)(Class, SEL)>(objc_msgSend)(SLPMetalLayerViewClass, sel_getUid("alloc"));
+
+	auto rect = (CGRect){{0, 0}, {frame.size.width, frame.size.height}};
+	id metal_view = reinterpret_cast<id (*)(id, SEL, CGRect)>(objc_msgSend)(alloc, sel_getUid("initWithFrame:"), rect);
+	objc_msgSend(metal_view, sel_getUid("setWantsLayer:"), YES);
+
+	// The below does: objc_msgSend(view, sel_getUid("setAutoresizingMask"), NSViewWidthSizable | NSViewHeightSizable);
+	// All this is doing is telling the view/layer to resize when the parent does.
+	objc_msgSend(metal_view, sel_getUid("setAutoresizingMask:"), 18);
+
+	objc_msgSend(view, sel_getUid("addSubview:"), metal_view);
+	s_metal_view_handle = metal_view;
 #endif
 }
 }

Source/Core/VideoCommon/RenderBase.h

@@ -149,6 +149,7 @@ class Renderer
 	// Final surface changing
 	// This is called when the surface is resized (WX) or the window changes (Android).
 	virtual void ChangeSurface(void* new_surface_handle) {}
+	virtual void CacheSurfaceHandle(void* new_surface_handle) {}
 	bool UseVertexDepthRange() const;
 protected:
 	std::tuple<int, int> CalculateTargetScale(int x, int y) const;
@@ -188,6 +189,7 @@ class Renderer
 	Common::Flag m_surface_needs_change;
 	Common::Event m_surface_changed;
 	void* m_new_surface_handle = nullptr;
+	void* m_cached_surface_handle = nullptr;
 private:
 	void RunFrameDumps();
 	void ShutdownFrameDumping();
@@ -231,4 +233,4 @@ class Renderer
 
 };
 
-extern std::unique_ptr<Renderer> g_renderer;
+extern std::unique_ptr<Renderer> g_renderer;