Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Linking native libraries into superhost #38684

Merged
merged 20 commits into from
Jul 16, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 9 additions & 13 deletions src/coreclr/src/pal/src/loader/module.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,13 @@ PAL_LoadLibraryDirect(
lpLibFileName ? lpLibFileName : W16_NULLSTRING,
lpLibFileName ? lpLibFileName : W16_NULLSTRING);

// Getting nullptr as name indicates redirection to current library
if (lpLibFileName == nullptr)
{
dl_handle = dlopen(NULL, RTLD_LAZY);
goto done;
}

if (!LOADVerifyLibraryPath(lpLibFileName))
{
goto done;
Expand Down Expand Up @@ -1436,18 +1443,6 @@ static LPWSTR LOADGetModuleFileName(MODSTRUCT *module)
return module->lib_name;
}

static bool ShouldRedirectToCurrentLibrary(LPCSTR libraryNameOrPath)
{
if (!g_running_in_exe)
return false;

// Getting nullptr as name indicates redirection to current library
if (libraryNameOrPath == nullptr)
return true;

return false;
}

/*
Function:
LOADLoadLibraryDirect [internal]
Expand All @@ -1464,7 +1459,8 @@ static NATIVE_LIBRARY_HANDLE LOADLoadLibraryDirect(LPCSTR libraryNameOrPath)
{
NATIVE_LIBRARY_HANDLE dl_handle;

if (ShouldRedirectToCurrentLibrary(libraryNameOrPath))
// Getting nullptr as name indicates redirection to current library
if (libraryNameOrPath == nullptr)
{
dl_handle = dlopen(NULL, RTLD_LAZY);
}
Expand Down
25 changes: 25 additions & 0 deletions src/coreclr/src/vm/dllimport.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@
#include "clr/fs/path.h"
using namespace clr::fs;

// Specifies whether coreclr is embedded or standalone
extern bool g_coreclr_embedded;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where are we setting this variable? I cannot see it in this PR.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the variable was introduced when we did coreclr embedding (previous PR of this series). It relies on coreclr mergeable knowing this fact at compile time.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// g_coreclr_embedded indicates that coreclr is linked directly into the program

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, thanks for the details, I've missed this one, I somehow thought we had just the g_running_in_exe.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

g_running_in_exe is the PAL mirror of g_coreclr_embedded, since it can't see g_coreclr_embedded directly.


// remove when we get an updated SDK
#define LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR 0x00000100
#define LOAD_LIBRARY_SEARCH_DEFAULT_DIRS 0x00001000
Expand Down Expand Up @@ -6303,6 +6306,28 @@ namespace
}
#endif // FEATURE_CORESYSTEM && !TARGET_UNIX

#if defined(TARGET_LINUX)
if (g_coreclr_embedded)
{
// this matches exactly the names in Interop.Libraries.cs
static const LPCWSTR toRedirect[] = {
W("libSystem.Native"),
W("libSystem.IO.Compression.Native"),
W("libSystem.Net.Security.Native"),
W("libSystem.Security.Cryptography.Native.OpenSsl")
};

int count = lengthof(toRedirect);
for (int i = 0; i < count; ++i)
{
if (wcscmp(wszLibName, toRedirect[i]) == 0)
{
return PAL_LoadLibraryDirect(NULL);
}
}
}
#endif

AppDomain* pDomain = GetAppDomain();
DWORD loadWithAlteredPathFlags = GetLoadWithAlteredSearchPathFlag();
bool libNameIsRelativePath = Path::IsRelative(wszLibName);
Expand Down
3 changes: 2 additions & 1 deletion src/installer/corehost/Windows/gen-buildsys-win.bat
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ popd
:DoGen
set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_SYSTEM_VERSION=10.0" "-DCLI_CMAKE_HOST_VER=%__HostVersion%" "-DCLI_CMAKE_COMMON_HOST_VER=%__AppHostVersion%" "-DCLI_CMAKE_HOST_FXR_VER=%__HostFxrVersion%"
set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCLI_CMAKE_HOST_POLICY_VER=%__HostPolicyVersion%" "-DCLI_CMAKE_PKG_RID=%cm_BaseRid%" "-DCLI_CMAKE_COMMIT_HASH=%__LatestCommit%" "-DCLR_CMAKE_HOST_ARCH=%__Arch%"
set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_INSTALL_PREFIX=%__CMakeBinDir%" "-DCLI_CMAKE_RESOURCE_DIR=%__ResourcesDir%" "-DCLR_ENG_NATIVE_DIR="%__sourceDir%\..\..\..\eng\native"
set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCORECLR_ARTIFACTS=%__CoreClrArtifacts% " "-DNATIVE_LIBS_ARTIFACTS=%__NativeLibsArtifacts%"
set __ExtraCmakeParams=%__ExtraCmakeParams% "-DCMAKE_INSTALL_PREFIX=%__CMakeBinDir%" "-DCLI_CMAKE_RESOURCE_DIR=%__ResourcesDir%" "-DCLR_ENG_NATIVE_DIR=%__sourceDir%\..\..\..\eng\native"

echo "%CMakePath%" %__sourceDir% -G "Visual Studio %__VSString%" %__ExtraCmakeParams%
"%CMakePath%" %__sourceDir% -G "Visual Studio %__VSString%" %__ExtraCmakeParams%
Expand Down
1 change: 1 addition & 0 deletions src/installer/corehost/build.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ if /i [%1] == [commit] (set __CommitSha=%2&&shift&&shift&goto Arg_Loop)
if /i [%1] == [incremental-native-build] ( set __IncrementalNativeBuild=1&&shift&goto Arg_Loop)
if /i [%1] == [rootDir] ( set __rootDir=%2&&shift&&shift&goto Arg_Loop)
if /i [%1] == [coreclrartifacts] (set __CoreClrArtifacts=%2&&shift&&shift&goto Arg_Loop)
if /i [%1] == [nativelibsartifacts] (set __NativeLibsArtifacts=%2&&shift&&shift&goto Arg_Loop)


shift
Expand Down
2 changes: 2 additions & 0 deletions src/installer/corehost/build.proj
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
<BuildArgs Condition="'$(Compiler)' != ''">$(BuildArgs) $(Compiler)</BuildArgs>
<BuildArgs Condition="'$(CMakeArgs)' != ''">$(BuildArgs) $(CMakeArgs)</BuildArgs>
<BuildArgs>$(BuildArgs) -coreclrartifacts $(CoreCLRArtifactsPath)</BuildArgs>
<BuildArgs>$(BuildArgs) -nativelibsartifacts $(LibrariesArtifactsPath)/bin/native/$(NetCoreAppCurrent)-$(TargetOS)-$(LibrariesConfiguration)-$(TargetArchitecture)</BuildArgs>
</PropertyGroup>

<!--
Expand Down Expand Up @@ -87,6 +88,7 @@
<BuildArgs Condition="'$(IncrementalNativeBuild)' == 'true'">$(BuildArgs) incremental-native-build</BuildArgs>
<BuildArgs>$(BuildArgs) rootdir $(RepoRoot)</BuildArgs>
<BuildArgs>$(BuildArgs) coreclrartifacts $(CoreCLRArtifactsPath)</BuildArgs>
<BuildArgs>$(BuildArgs) nativelibsartifacts $(LibrariesArtifactsPath)/bin/native/$(NetCoreAppCurrent)-$(TargetOS)-$(LibrariesConfiguration)-$(TargetArchitecture)</BuildArgs>
</PropertyGroup>

<!--
Expand Down
8 changes: 7 additions & 1 deletion src/installer/corehost/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@ handle_arguments() {
__ShiftArgs=1
;;

nativelibsartifacts|-nativelibsartifacts)
__NativeLibsArtifacts="$2"
__ShiftArgs=1
;;

*)
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
esac
Expand All @@ -82,10 +87,11 @@ __DistroRidLower="$(echo $__DistroRid | tr '[:upper:]' '[:lower:]')"
__BinDir="$__RootBinDir/bin/$__DistroRidLower.$__BuildType"
__IntermediatesDir="$__RootBinDir/obj/$__DistroRidLower.$__BuildType"

export __BinDir __IntermediatesDir __CoreClrArtifacts
export __BinDir __IntermediatesDir __CoreClrArtifacts __NativeLibsArtifacts

__CMakeArgs="-DCLI_CMAKE_HOST_VER=\"$__host_ver\" -DCLI_CMAKE_COMMON_HOST_VER=\"$__apphost_ver\" -DCLI_CMAKE_HOST_FXR_VER=\"$__fxr_ver\" $__CMakeArgs"
__CMakeArgs="-DCLI_CMAKE_HOST_POLICY_VER=\"$__policy_ver\" -DCLI_CMAKE_PKG_RID=\"$__DistroRid\" -DCLI_CMAKE_COMMIT_HASH=\"$__commit_hash\" $__CMakeArgs"
VSadov marked this conversation as resolved.
Show resolved Hide resolved
__CMakeArgs="-DCORECLR_ARTIFACTS=\"$__CoreClrArtifacts\" -DNATIVE_LIBS_ARTIFACTS=\"$__NativeLibsArtifacts\" $__CMakeArgs"

if [[ "$__PortableBuild" == 1 ]]; then
__CMakeArgs="-DCLI_CMAKE_PORTABLE_BUILD=1 $__CMakeArgs"
Expand Down
159 changes: 114 additions & 45 deletions src/installer/corehost/cli/apphost/static/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ if(CLR_CMAKE_TARGET_WIN32)
endif()

include(../../exe.cmake)
include(configure.cmake)

add_definitions(-DFEATURE_APPHOST=1)
add_definitions(-DFEATURE_STATIC_HOST=1)
Expand All @@ -57,7 +58,7 @@ endif()

# Path like: artifacts/bin/coreclr/Windows_NT.x64.Release/lib or
# /root/runtime/artifacts/transport/coreclr/lib
set(CORECLR_STATIC_LIB_LOCATION "$ENV{__CoreClrArtifacts}/lib")
set(CORECLR_STATIC_LIB_LOCATION "${CORECLR_ARTIFACTS}/lib")

message ("Looking for coreclr_static lib at location: '${CORECLR_STATIC_LIB_LOCATION}'.")

Expand All @@ -78,62 +79,122 @@ if(CLR_CMAKE_TARGET_WIN32)
bcrypt.lib
RuntimeObject.lib
)
else()
elseif(CLR_CMAKE_TARGET_LINUX)
set(CORECLR_LIBRARIES
${CORECLR_STATIC_LIB_LOCATION}/libcoreclr_static.a
${CORECLR_STATIC_LIB_LOCATION}/libcoreclrpal.a
${CORECLR_STATIC_LIB_LOCATION}/libpalrt.a
${CORECLR_STATIC_LIB_LOCATION}/libeventprovider.a
${CORECLR_STATIC_LIB_LOCATION}/libnativeresourcestring.a
)
endif(CLR_CMAKE_TARGET_WIN32)

if(CLR_CMAKE_TARGET_OSX)
find_library(COREFOUNDATION CoreFoundation)
find_library(CORESERVICES CoreServices)
find_library(SECURITY Security)
find_library(SYSTEM System)

LIST(APPEND CORECLR_LIBRARIES
${COREFOUNDATION}
${CORESERVICES}
${SECURITY}
${SYSTEM}
)
endif(CLR_CMAKE_TARGET_OSX)

# On OSX and *BSD, we use the libunwind that's part of the OS
if(CLR_CMAKE_TARGET_FREEBSD)
find_unwind_libs(UNWIND_LIBS)

LIST(APPEND CORECLR_LIBRARIES
${UNWIND_LIBS}
)
endif(CLR_CMAKE_TARGET_FREEBSD)
# currently linking coreclr into the singlefilehost is only supported on linux
# the following code here would be needed if/when BSD and OSX are supported too
#
# if(CLR_CMAKE_TARGET_OSX)
# find_library(COREFOUNDATION CoreFoundation)
# find_library(CORESERVICES CoreServices)
# find_library(SECURITY Security)
# find_library(SYSTEM System)
#
# LIST(APPEND CORECLR_LIBRARIES
# ${COREFOUNDATION}
# ${CORESERVICES}
# ${SECURITY}
# ${SYSTEM}
# )
# endif(CLR_CMAKE_TARGET_OSX)
#
# # On OSX and *BSD, we use the libunwind that's part of the OS
# if(CLR_CMAKE_TARGET_FREEBSD)
# find_unwind_libs(UNWIND_LIBS)
#
# LIST(APPEND CORECLR_LIBRARIES
# ${UNWIND_LIBS}
# )
# endif(CLR_CMAKE_TARGET_FREEBSD)
#
# if(CLR_CMAKE_TARGET_NETBSD)
# find_library(KVM kvm)
#
# LIST(APPEND CORECLR_LIBRARIES
# ${KVM}
# )
# endif(CLR_CMAKE_TARGET_NETBSD)
endif(CLR_CMAKE_TARGET_WIN32)

if(CLR_CMAKE_TARGET_NETBSD)
find_library(KVM kvm)
# Path like: artifacts/bin/native/net5.0-Linux-Release-arm/
set(NATIVE_LIBS_LOCATION "${NATIVE_LIBS_ARTIFACTS}")
message ("Looking for native libs at location: '${NATIVE_LIBS_LOCATION}'.")

LIST(APPEND CORECLR_LIBRARIES
${KVM}
if(NOT CLR_CMAKE_TARGET_LINUX)
set(NATIVE_LIBS
# Native libs linked into singlefilehost is supported only on Linux for now.
# if/when BSD and OSX are supported too, consider the commented code sections below.
)
else()
set(NATIVE_LIBS
${NATIVE_LIBS_LOCATION}/libSystem.IO.Compression.Native.a
${NATIVE_LIBS_LOCATION}/libSystem.Native.a
${NATIVE_LIBS_LOCATION}/libSystem.Net.Security.Native.a
${NATIVE_LIBS_LOCATION}/libSystem.Security.Cryptography.Native.OpenSsl.a
)
endif(CLR_CMAKE_TARGET_NETBSD)

if(NOT CLR_CMAKE_TARGET_WIN32)
set(DEF_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/singlefilehost_unixexports.src)
set(EXPORTS_FILE ${CMAKE_CURRENT_BINARY_DIR}/singlefilehost.exports)
generate_exports_file(${DEF_SOURCES} ${EXPORTS_FILE})

set_exports_linker_option(${EXPORTS_FILE})
endif(NOT CLR_CMAKE_TARGET_WIN32)

if(CLR_CMAKE_HOST_UNIX)
add_custom_target(singlefilehost_exports DEPENDS ${EXPORTS_FILE})
add_dependencies(singlefilehost singlefilehost_exports)

set_property(TARGET singlefilehost APPEND_STRING PROPERTY LINK_FLAGS ${EXPORTS_LINKER_OPTION})
set_property(TARGET singlefilehost APPEND_STRING PROPERTY LINK_DEPENDS ${EXPORTS_FILE})
endif(CLR_CMAKE_HOST_UNIX)
find_package(ZLIB REQUIRED)

# Additional requirements for System.System.IO.Compression.Native
#
# if (CLR_CMAKE_TARGET_SUNOS)
# set(ZLIB_LIBRARIES z m)
# elseif (CLR_CMAKE_TARGET_UNIX)
# find_package(ZLIB REQUIRED)
# endif ()

# Additional requirements for System.Net.Security.Native
if (HAVE_GSSFW_HEADERS)
find_library(LIBGSS NAMES GSS)
if(LIBGSS STREQUAL LIBGSS-NOTFOUND)
message(FATAL_ERROR "Cannot find GSS.Framework and System.Net.Security.Native cannot build without it. Try installing GSS.Framework (or the appropriate package for your platform)")
endif()
elseif(HAVE_HEIMDAL_HEADERS)
find_library(LIBGSS NAMES gssapi)
if(LIBGSS STREQUAL LIBGSS-NOTFOUND)
message(FATAL_ERROR "Cannot find libgssapi and System.Net.Security.Native cannot build without it. Try installing heimdal (or the appropriate package for your platform)")
endif()
else()
find_library(LIBGSS NAMES gssapi_krb5)
if(LIBGSS STREQUAL LIBGSS-NOTFOUND)
message(FATAL_ERROR "Cannot find libgssapi_krb5 and System.Net.Security.Native cannot build without it. Try installing libkrb5-dev (or the appropriate package for your platform)")
endif()
endif()

# Additional requirements for System.Native
if (CLR_CMAKE_TARGET_LINUX AND NOT CLR_CMAKE_TARGET_ANDROID)
set(NATIVE_LIBS_EXTRA
rt
)
# elseif (CLR_CMAKE_TARGET_FREEBSD)
# set(NATIVE_LIBS_EXTRA
# pthread
# )
# find_library(INOTIFY_LIBRARY inotify HINTS /usr/local/lib)
# if(NOT (INOTIFY_LIBRARY STREQUAL INOTIFY_LIBRARY-NOTFOUND))
# LIST(APPEND NATIVE_LIBS_EXTRA
# ${INOTIFY_LIBRARY}
# )
# endif ()
# elseif (CLR_CMAKE_TARGET_SUNOS)
# set(NATIVE_LIBS_EXTRA
# socket
# )
endif ()

if(CLR_CMAKE_TARGET_LINUX OR CLR_CMAKE_TARGET_FREEBSD OR CLR_CMAKE_TARGET_NETBSD OR CLR_CMAKE_TARGET_SUNOS)
# These options are used to force every object to be included even if it's unused.
set(START_WHOLE_ARCHIVE -Wl,--whole-archive)
set(END_WHOLE_ARCHIVE -Wl,--no-whole-archive)
endif(CLR_CMAKE_TARGET_LINUX OR CLR_CMAKE_TARGET_FREEBSD OR CLR_CMAKE_TARGET_NETBSD OR CLR_CMAKE_TARGET_SUNOS)
endif(NOT CLR_CMAKE_TARGET_LINUX)

set_property(TARGET singlefilehost PROPERTY ENABLE_EXPORTS 1)

Expand All @@ -142,4 +203,12 @@ target_link_libraries(singlefilehost
libhostpolicy_static
libhostcommon
${CORECLR_LIBRARIES}

${ZLIB_LIBRARIES}
${LIBGSS}
${NATIVE_LIBS_EXTRA}

${START_WHOLE_ARCHIVE}
${NATIVE_LIBS}
${END_WHOLE_ARCHIVE}
)
12 changes: 12 additions & 0 deletions src/installer/corehost/cli/apphost/static/configure.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
include(CheckIncludeFiles)

check_include_files(
GSS/GSS.h
HAVE_GSSFW_HEADERS)

option(HeimdalGssApi "use heimdal implementation of GssApi" OFF)
if (HeimdalGssApi)
check_include_files(
gssapi/gssapi.h
HAVE_HEIMDAL_HEADERS)
endif()

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ extern "C"
}


#if defined(_WIN32)
#if !defined(TARGET_LINUX)

bool coreclr_resolver_t::resolve_coreclr(const pal::string_t& libcoreclr_path, coreclr_resolver_contract_t& coreclr_resolver_contract)
{
Expand Down
10 changes: 4 additions & 6 deletions src/installer/pkg/projects/netcoreapp/pkg/Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,17 @@
</PropertyGroup>

<ItemGroup>
<!-- UNIX -->
<SingleFileHostIncludeFilename Include="libSystem.IO.Compression.Native.so" />
<SingleFileHostIncludeFilename Include="libSystem.Native.so" />
<SingleFileHostIncludeFilename Include="libSystem.Net.Security.Native.so" />
<SingleFileHostIncludeFilename Include="libSystem.Security.Cryptography.Native.OpenSsl.so" />
<!-- LINUX -->

<!-- OSX -->
<SingleFileHostIncludeFilename Include="libSystem.IO.Compression.Native.dylib" />
<SingleFileHostIncludeFilename Include="libSystem.Native.dylib" />
<SingleFileHostIncludeFilename Include="libSystem.Net.Security.Native.dylib" />
<SingleFileHostIncludeFilename Include="libSystem.Security.Cryptography.Native.Apple.dylib" />
<SingleFileHostIncludeFilename Include="libSystem.Security.Cryptography.Native.OpenSsl.dylib" />

<SingleFileHostIncludeFilename Include="libclrjit.dylib" />
<SingleFileHostIncludeFilename Include="libcoreclr.dylib" />

<!-- Windows -->
<SingleFileHostIncludeFilename Include="clrcompression.dll" />
<SingleFileHostIncludeFilename Include="clrjit.dll" />
Expand Down