System: Add advanced 'Export Shared Memory' option

Memory map is exported as duckstation_<pid>. Previously, this only
worked on Windows, now it is extended to Linux as well.

src/common/crash_handler.cpp

@@ -85,6 +85,7 @@ static bool WriteMinidump(HMODULE hDbgHelp, HANDLE hFile, HANDLE hProcess, DWORD
 
 static std::wstring s_write_directory;
 static DynamicLibrary s_dbghelp_module;
+static CrashHandler::CleanupHandler s_cleanup_handler;
 static bool s_in_crash_handler = false;
 
 static void GenerateCrashFilename(wchar_t* buf, size_t len, const wchar_t* prefix, const wchar_t* extension)
@@ -99,8 +100,6 @@ static void GenerateCrashFilename(wchar_t* buf, size_t len, const wchar_t* prefi
 
 static void WriteMinidumpAndCallstack(PEXCEPTION_POINTERS exi)
 {
-  s_in_crash_handler = true;
-
   wchar_t filename[1024] = {};
   GenerateCrashFilename(filename, std::size(filename), s_write_directory.empty() ? nullptr : s_write_directory.c_str(),
                         L"txt");
@@ -148,15 +147,21 @@ static LONG NTAPI ExceptionHandler(PEXCEPTION_POINTERS exi)
 {
   // if the debugger is attached, or we're recursively crashing, let it take care of it.
   if (!s_in_crash_handler)
+  {
+    s_in_crash_handler = true;
+    if (s_cleanup_handler)
+      s_cleanup_handler();
+
     WriteMinidumpAndCallstack(exi);
+  }
 
   // returning EXCEPTION_CONTINUE_SEARCH makes sense, except for the fact that it seems to leave zombie processes
   // around. instead, force ourselves to terminate.
   TerminateProcess(GetCurrentProcess(), 0xFEFEFEFEu);
   return EXCEPTION_CONTINUE_SEARCH;
 }
 
-bool CrashHandler::Install()
+bool CrashHandler::Install(CleanupHandler cleanup_handler)
 {
   // load dbghelp at install/startup, that way we're not LoadLibrary()'ing after a crash
   // .. because that probably wouldn't go down well.
@@ -165,6 +170,7 @@ bool CrashHandler::Install()
     s_dbghelp_module.Adopt(mod);
 
   SetUnhandledExceptionFilter(ExceptionHandler);
+  s_cleanup_handler = cleanup_handler;
   return true;
 }
 
@@ -208,6 +214,7 @@ static void LogCallstack(int signal, const void* exception_pc);
 static std::recursive_mutex s_crash_mutex;
 static bool s_in_signal_handler = false;
 
+static CleanupHandler s_cleanup_handler;
 static backtrace_state* s_backtrace_state = nullptr;
 } // namespace CrashHandler
 
@@ -304,6 +311,9 @@ void CrashHandler::CrashSignalHandler(int signal, siginfo_t* siginfo, void* ctx)
   {
     s_in_signal_handler = true;
 
+    if (s_cleanup_handler)
+      s_cleanup_handler();
+
 #if defined(__APPLE__) && defined(__x86_64__)
     void* const exception_pc = reinterpret_cast<void*>(static_cast<ucontext_t*>(ctx)->uc_mcontext->__ss.__rip);
 #elif defined(__FreeBSD__) && defined(__x86_64__)
@@ -327,7 +337,7 @@ void CrashHandler::CrashSignalHandler(int signal, siginfo_t* siginfo, void* ctx)
   std::abort();
 }
 
-bool CrashHandler::Install()
+bool CrashHandler::Install(CleanupHandler cleanup_handler)
 {
   const std::string progpath = FileSystem::GetProgramPath();
   s_backtrace_state = backtrace_create_state(progpath.empty() ? nullptr : progpath.c_str(), 0, nullptr, nullptr);
@@ -344,6 +354,7 @@ bool CrashHandler::Install()
   if (sigaction(SIGSEGV, &sa, nullptr) != 0)
     return false;
 
+  s_cleanup_handler = cleanup_handler;
   return true;
 }
 
@@ -358,7 +369,7 @@ void CrashHandler::WriteDumpForCaller()
 
 #else
 
-bool CrashHandler::Install()
+bool CrashHandler::Install(CleanupHandler cleanup_handler)
 {
   return false;
 }

src/common/crash_handler.h

@@ -1,21 +1,30 @@
 // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
-#include "types.h"
+#pragma once
+
 #include <string_view>
 
 #ifndef _WIN32
 #include <signal.h>
 #endif
 
 namespace CrashHandler {
-bool Install();
+
+/// Adds a callback to run just before the crash handler exits.
+/// It's not guaranteed that this handler will actually run, because the process state could be very messed up by this
+/// point. It's mainly a thing so that we can free up the shared memory object if there was one created.
+using CleanupHandler = void(*)();
+
+bool Install(CleanupHandler cleanup_handler);
 void SetWriteDirectory(std::string_view dump_directory);
 void WriteDumpForCaller();
 
 #ifndef _WIN32
+
 // Allow crash handler to be invoked from a signal.
 void CrashSignalHandler(int signal, siginfo_t* siginfo, void* ctx);
+
 #endif
 
 } // namespace CrashHandler

src/common/memmap.cpp

@@ -66,9 +66,10 @@ std::string MemMap::GetFileMappingName(const char* prefix)
 
 void* MemMap::CreateSharedMemory(const char* name, size_t size, Error* error)
 {
+  const std::wstring mapping_name = name ? StringUtil::UTF8StringToWideString(name) : std::wstring();
   const HANDLE mapping =
     CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, static_cast<DWORD>(size >> 32),
-                       static_cast<DWORD>(size), StringUtil::UTF8StringToWideString(name).c_str());
+                       static_cast<DWORD>(size), mapping_name.empty() ? nullptr : mapping_name.c_str());
   if (!mapping)
     Error::SetWin32(error, "CreateFileMappingW() failed: ", GetLastError());
 
@@ -80,6 +81,11 @@ void MemMap::DestroySharedMemory(void* ptr)
   CloseHandle(static_cast<HANDLE>(ptr));
 }
 
+void MemMap::DeleteSharedMemory(const char* name)
+{
+  // Automatically freed on close.
+}
+
 void* MemMap::MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode)
 {
   void* ret = MapViewOfFileEx(static_cast<HANDLE>(handle), FILE_MAP_READ | FILE_MAP_WRITE,
@@ -374,6 +380,10 @@ void MemMap::DestroySharedMemory(void* ptr)
   mach_port_deallocate(mach_task_self(), static_cast<mach_port_t>(reinterpret_cast<uintptr_t>(ptr)));
 }
 
+void MemMap::DeleteSharedMemory(const char* name)
+{
+}
+
 void* MemMap::MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode)
 {
   mach_vm_address_t ptr = reinterpret_cast<mach_vm_address_t>(baseaddr);
@@ -617,15 +627,25 @@ std::string MemMap::GetFileMappingName(const char* prefix)
 
 void* MemMap::CreateSharedMemory(const char* name, size_t size, Error* error)
 {
+  const bool is_anonymous = (!name || *name == 0);
+#if defined(__linux__) || defined(__FreeBSD__)
+  const int fd = is_anonymous ? memfd_create("", 0) : shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600);
+  if (fd < 0)
+  {
+    Error::SetErrno(error, is_anonymous ? "memfd_create() failed: " : "shm_open() failed: ", errno);
+    return nullptr;
+  }
+#else
   const int fd = shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600);
   if (fd < 0)
   {
-    Error::SetErrno(error, "shm_open failed: ", errno);
+    Error::SetErrno(error, "shm_open() failed: ", errno);
     return nullptr;
   }
 
   // we're not going to be opening this mapping in other processes, so remove the file
   shm_unlink(name);
+#endif
 
   // use fallocate() to ensure we don't SIGBUS later on.
 #ifdef __linux__
@@ -651,6 +671,11 @@ void MemMap::DestroySharedMemory(void* ptr)
   close(static_cast<int>(reinterpret_cast<intptr_t>(ptr)));
 }
 
+void MemMap::DeleteSharedMemory(const char* name)
+{
+  shm_unlink(name);
+}
+
 void* MemMap::MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode)
 {
   const int flags = (baseaddr != nullptr) ? (MAP_SHARED | MAP_FIXED) : MAP_SHARED;

src/common/memmap.h

@@ -53,6 +53,7 @@ class Error;
 namespace MemMap {
 std::string GetFileMappingName(const char* prefix);
 void* CreateSharedMemory(const char* name, size_t size, Error* error);
+void DeleteSharedMemory(const char* name);
 void DestroySharedMemory(void* ptr);
 void* MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode);
 void UnmapSharedMemory(void* baseaddr, size_t size);