Library Forwarding: Make building on weak cross-compiling setups easier

CMakeLists.txt

@@ -7,8 +7,9 @@ CHECK_INCLUDE_FILES ("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
 option(BUILD_TESTS "Build unit tests to ensure sanity" TRUE)
 option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests, requires x86 compiler" FALSE)
 option(BUILD_THUNKS "Build thunks" FALSE)
-option(BUILD_FEXCONFIG "Build FEXConfig, requires SDL2 and X11" TRUE)
 option(ENABLE_CLANG_THUNKS "Build thunks with clang" FALSE)
+option(BUILD_GUEST_LIBS "Build guest library wrappers as part of the main project" TRUE)
+option(BUILD_FEXCONFIG "Build FEXConfig, requires SDL2 and X11" TRUE)
 option(ENABLE_IWYU "Enables include what you use program" FALSE)
 option(ENABLE_LTO "Enable LTO with compilation" TRUE)
 option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
@@ -34,6 +35,7 @@ option(ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT "Enables glibc memory allocation hookin
 
 set (X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
 set (X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
+set (X86_DEV_ROOTFS "/" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
 set (DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu" CACHE PATH "global data directory")
 
 string(FIND ${CMAKE_BASE_NAME} mingw CONTAINS_MINGW)
@@ -381,83 +383,7 @@ if (BUILD_TESTS)
 endif()
 
 if (BUILD_THUNKS)
-  set (FEX_PROJECT_SOURCE_DIR ${PROJECT_SOURCE_DIR})
-  add_subdirectory(ThunkLibs/Generator)
-
-  # Thunk targets for both host libraries and IDE integration
-  add_subdirectory(ThunkLibs/HostLibs)
-
-  # Thunk targets for IDE integration of guest code, only
-  add_subdirectory(ThunkLibs/GuestLibs)
-
-  # Thunk targets for guest libraries
-  include(ExternalProject)
-  ExternalProject_Add(guest-libs
-    PREFIX guest-libs
-    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/GuestLibs"
-    BINARY_DIR "Guest"
-    CMAKE_ARGS
-      "-DBITNESS=64"
-      "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
-      "-DBUILD_FEX_LINUX_TESTS=${BUILD_FEX_LINUX_TESTS}"
-      "-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}"
-      "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_64_TOOLCHAIN_FILE}"
-      "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
-      "-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}"
-      "-DGENERATOR_EXE=$<TARGET_FILE:thunkgen>"
-    INSTALL_COMMAND ""
-    BUILD_ALWAYS ON
-    DEPENDS thunkgen
-  )
-
-  ExternalProject_Add(guest-libs-32
-    PREFIX guest-libs-32
-    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/GuestLibs"
-    BINARY_DIR "Guest_32"
-    CMAKE_ARGS
-      "-DBITNESS=32"
-      "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
-      "-DBUILD_FEX_LINUX_TESTS=${BUILD_FEX_LINUX_TESTS}"
-      "-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}"
-      "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_32_TOOLCHAIN_FILE}"
-      "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
-      "-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}"
-      "-DGENERATOR_EXE=$<TARGET_FILE:thunkgen>"
-    INSTALL_COMMAND ""
-    BUILD_ALWAYS ON
-    DEPENDS thunkgen
-  )
-
-  install(
-    CODE "MESSAGE(\"-- Installing: guest-libs\")"
-    CODE "
-    EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
-    )"
-    DEPENDS guest-libs
-  )
-
-  install(
-    CODE "MESSAGE(\"-- Installing: guest-libs-32\")"
-    CODE "
-    EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
-    )"
-    DEPENDS guest-libs-32
-  )
-
-  add_custom_target(uninstall_guest-libs
-    COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
-  )
-
-  add_custom_target(uninstall_guest-libs-32
-    COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
-  )
-
-  add_dependencies(uninstall uninstall_guest-libs)
-  add_dependencies(uninstall uninstall_guest-libs-32)
+  add_subdirectory(ThunkLibs)
 endif()
 
 set(FEX_VERSION_MAJOR "0")

ThunkLibs/CMakeLists.txt

@@ -0,0 +1,108 @@
+cmake_minimum_required(VERSION 3.14)
+project(FEXLibWrappers)
+
+set(CMAKE_CXX_STANDARD 20)
+set(FEX_PROJECT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/..)
+
+add_subdirectory(Generator)
+
+# Two methods of building guest library wrappers are supported:
+# - build as part of the main project (recommended)
+# - build in isolation from a dedicated build folder
+# The second method is useful when it's difficult to set up a full
+# cross-compiling environment targeting x86. In those cases, guest library
+# compilation can be moved to a different machine / VM.
+if (NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+  # When included during a regular FEX build, build both guest and host libs
+  # unless requested otherwise
+  set(BUILD_HOST_LIBS TRUE)
+  # BUILD_GUEST_LIBS from parent project
+
+  # However, include targets for IDE integration of guest libs
+  add_subdirectory(GuestLibs)
+else()
+  # When included on our own, only build guest libraries
+  set(BUILD_HOST_LIBS FALSE)
+  set(BUILD_GUEST_LIBS TRUE)
+endif()
+
+if (BUILD_HOST_LIBS)
+  # Targets for building host libraries and for IDE integration
+  add_subdirectory(HostLibs)
+endif()
+
+if (BUILD_GUEST_LIBS)
+  # Targets for building guest libraries
+  include(ExternalProject)
+
+  ExternalProject_Add(guest-libs
+    PREFIX Guest/guest-libs
+    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/GuestLibs"
+    BINARY_DIR "../Guest"
+    CMAKE_ARGS
+      "-DBITNESS=64"
+      "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+      "-DBUILD_FEX_LINUX_TESTS=${BUILD_FEX_LINUX_TESTS}"
+      "-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}"
+      "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${FEX_PROJECT_SOURCE_DIR}/toolchain_x86_64.cmake"
+      "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
+      "-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}"
+      "-DGENERATOR_EXE=$<TARGET_FILE:thunkgen>"
+      "-DX86_DEV_ROOTFS=${X86_DEV_ROOTFS}"
+    INSTALL_COMMAND ""
+    BUILD_ALWAYS ON
+    DEPENDS thunkgen
+  )
+
+  ExternalProject_Add(guest-libs-32
+    PREFIX Guest_32/guest-libs-32
+    SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/GuestLibs"
+    BINARY_DIR "../Guest_32"
+    CMAKE_ARGS
+      "-DBITNESS=32"
+      "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+      "-DBUILD_FEX_LINUX_TESTS=${BUILD_FEX_LINUX_TESTS}"
+      "-DENABLE_CLANG_THUNKS=${ENABLE_CLANG_THUNKS}"
+      "-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${FEX_PROJECT_SOURCE_DIR}/toolchain_x86_32.cmake"
+      "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
+      "-DFEX_PROJECT_SOURCE_DIR=${FEX_PROJECT_SOURCE_DIR}"
+      "-DGENERATOR_EXE=$<TARGET_FILE:thunkgen>"
+      "-DX86_DEV_ROOTFS=${X86_DEV_ROOTFS}"
+    INSTALL_COMMAND ""
+    BUILD_ALWAYS ON
+    DEPENDS thunkgen
+  )
+
+  install(
+    CODE "MESSAGE(\"-- Installing: guest-libs\")"
+    CODE "
+    EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
+    )"
+    DEPENDS guest-libs
+  )
+
+  install(
+    CODE "MESSAGE(\"-- Installing: guest-libs-32\")"
+    CODE "
+    EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
+    )"
+    DEPENDS guest-libs-32
+  )
+
+  if (TARGET uninstall)
+    add_custom_target(uninstall_guest-libs
+      COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
+    )
+
+    add_custom_target(uninstall_guest-libs-32
+      COMMAND ${CMAKE_COMMAND} "--build" "." "--target" "uninstall"
+      WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest_32
+    )
+
+    add_dependencies(uninstall uninstall_guest-libs)
+    add_dependencies(uninstall uninstall_guest-libs-32)
+  endif ()
+endif()

ThunkLibs/Generator/CMakeLists.txt

@@ -1,3 +1,8 @@
+if (NOT TARGET fmt)
+  set(FMT_INSTALL OFF)
+  add_subdirectory(../../External/fmt/ fmt)
+endif ()
+
 find_package(Clang REQUIRED CONFIG)
 find_package(OpenSSL REQUIRED COMPONENTS Crypto)
 

ThunkLibs/Generator/main.cpp

@@ -4,6 +4,7 @@
 #include "llvm/Support/Signals.h"
 
 #include <iostream>
+#include <optional>
 #include <string>
 
 #include "interface.h"
@@ -17,7 +18,7 @@ void print_usage(const char* program_name) {
 int main(int argc, char* const argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
-  if (argc < 5) {
+  if (argc < 6) {
     print_usage(argv[0]);
     return EXIT_FAILURE;
   }
@@ -32,7 +33,7 @@ int main(int argc, char* const argv[]) {
   }
 
   // Process arguments before the "--" separator
-  if (argc != 5 && argc != 6) {
+  if (argc != 6 && argc != 7) {
     print_usage(argv[0]);
     return EXIT_FAILURE;
   }
@@ -42,6 +43,7 @@ int main(int argc, char* const argv[]) {
   const std::string libname = *arg++;
   const std::string target_abi = *arg++;
   const std::string output_filename = *arg++;
+  const std::string x86_rootfs = *arg++;
 
   OutputFilenames output_filenames;
   if (target_abi == "-host") {
@@ -65,27 +67,52 @@ int main(int argc, char* const argv[]) {
 
   ClangTool GuestTool = Tool;
 
-  {
-    const bool is_32bit_guest = (argv[5] == std::string_view {"-for-32bit-guest"});
-    auto append_guest_args = [is_32bit_guest](const clang::tooling::CommandLineArguments& Args, clang::StringRef) {
-      clang::tooling::CommandLineArguments AdjustedArgs = Args;
-      const char* platform = is_32bit_guest ? "i686" : "x86_64";
-      if (is_32bit_guest) {
-        AdjustedArgs.push_back("-m32");
-        AdjustedArgs.push_back("-DIS_32BIT_THUNK");
-      }
-      AdjustedArgs.push_back(std::string {"--target="} + platform + "-linux-unknown");
-      AdjustedArgs.push_back("-isystem");
-      AdjustedArgs.push_back(std::string {"/usr/"} + platform + "-linux-gnu/include/");
-      AdjustedArgs.push_back("-DGUEST_THUNK_LIBRARY");
-      return AdjustedArgs;
-    };
-    GuestTool.appendArgumentsAdjuster(append_guest_args);
-  }
-
+  auto append_x86_rootfs_includes = [&x86_rootfs](clang::tooling::CommandLineArguments& Args, const char* triple) {
+    if (x86_rootfs == "/") {
+      return;
+    }
+
+    Args.push_back("--sysroot");
+    Args.push_back(x86_rootfs);
+
+    // The dev rootfs is only really needed for the standard library.
+    // Other libraries generally don't have platform specific headers.
+    Args.push_back("-idirafter");
+    Args.push_back("/usr/include/");
+  };
+
+  // Analyse data layout for guest ABI
+  const bool is_32bit_guest = (argv[6] == std::string_view {"-for-32bit-guest"});
+  GuestTool.appendArgumentsAdjuster([&](const clang::tooling::CommandLineArguments& Args, clang::StringRef) {
+    clang::tooling::CommandLineArguments AdjustedArgs = Args;
+    const char* platform = is_32bit_guest ? "i686-linux-gnu" : "x86_64-linux-gnu";
+    if (is_32bit_guest) {
+      AdjustedArgs.push_back("-m32");
+      AdjustedArgs.push_back("-DIS_32BIT_THUNK");
+    }
+    AdjustedArgs.push_back("-DGUEST_THUNK_LIBRARY");
+    AdjustedArgs.push_back(std::string {"--target="} + platform);
+    AdjustedArgs.push_back("-isystem");
+    AdjustedArgs.push_back(std::string {"/usr/"} + platform + "/include/");
+
+    append_x86_rootfs_includes(AdjustedArgs, platform);
+
+    return AdjustedArgs;
+  });
   auto data_layout_analysis_factory = std::make_unique<AnalyzeDataLayoutActionFactory>();
   GuestTool.run(data_layout_analysis_factory.get());
   auto& data_layout = data_layout_analysis_factory->GetDataLayout();
 
+  // Run generator for target ABI
+  Tool.appendArgumentsAdjuster([&](const clang::tooling::CommandLineArguments& Args, clang::StringRef) {
+    clang::tooling::CommandLineArguments AdjustedArgs = Args;
+    AdjustedArgs.push_back("-DIS_HOST_THUNKGEN_PASS");
+    if (target_abi == "-guest") {
+      const char* platform = is_32bit_guest ? "i686-linux-gnu" : "x86_64-linux-gnu";
+      append_x86_rootfs_includes(AdjustedArgs, platform);
+    }
+
+    return AdjustedArgs;
+  });
   return Tool.run(std::make_unique<GenerateThunkLibsActionFactory>(std::move(libname), std::move(output_filenames), data_layout).get());
 }

ThunkLibs/GuestLibs/CMakeLists.txt

@@ -9,6 +9,10 @@ if (ENABLE_CLANG_THUNKS)
   add_link_options(${LD_OVERRIDE})
 endif()
 
+if (NOT X86_DEV_ROOTFS)
+  message(FATAL_ERROR "X86_DEV_ROOTFS must be set (use \"/\" to ignore)")
+endif()
+
 find_program(CCACHE_PROGRAM ccache)
 if(CCACHE_PROGRAM)
   message(STATUS "CCache enabled for guest thunks")
@@ -77,7 +81,7 @@ function(generate NAME SOURCE_FILE)
     OUTPUT "${OUTFILE}"
     DEPENDS "${GENERATOR_EXE}"
     DEPENDS "${SOURCE_FILE}"
-    COMMAND "${GENERATOR_EXE}" "${SOURCE_FILE}" "${NAME}" "-guest" "${OUTFILE}" ${BITNESS_FLAGS} -- -std=c++20 ${BITNESS_FLAGS2}
+    COMMAND "${GENERATOR_EXE}" "${SOURCE_FILE}" "${NAME}" "-guest" "${OUTFILE}" "${X86_DEV_ROOTFS}" ${BITNESS_FLAGS} -- -std=c++20 ${BITNESS_FLAGS2}
       # Expand compile definitions to space-separated list of -D parameters
       "$<$<BOOL:${compile_prop}>:;-D$<JOIN:${compile_prop},;-D>>"
       # Expand include directories to space-separated list of -isystem parameters

ThunkLibs/HostLibs/CMakeLists.txt

@@ -52,7 +52,7 @@ function(generate NAME SOURCE_FILE GUEST_BITNESS)
     OUTPUT "${OUTFILE}"
     DEPENDS "${SOURCE_FILE}"
     DEPENDS thunkgen
-    COMMAND thunkgen "${SOURCE_FILE}" "${NAME}" "-host" "${OUTFILE}" ${BITNESS_FLAGS} -- -std=c++20
+    COMMAND thunkgen "${SOURCE_FILE}" "${NAME}" "-host" "${OUTFILE}" "${X86_DEV_ROOTFS}" ${BITNESS_FLAGS} -- -std=c++20
       # Expand compile definitions to space-separated list of -D parameters
       "$<$<BOOL:${compile_prop}>:;-D$<JOIN:${compile_prop},;-D>>"
       # Expand include directories to space-separated list of -isystem parameters