From 1b0dd045184c92acac0f652dfceb4afc0201e026 Mon Sep 17 00:00:00 2001 From: chen-zhiyu Date: Sat, 10 Oct 2020 12:07:01 +0800 Subject: [PATCH] add musl option --- CMakeLists.txt | 1 + cmake/configure.cmake | 10 ++++++++++ paddle/fluid/inference/CMakeLists.txt | 2 +- paddle/fluid/inference/check_symbol.sh | 2 +- paddle/fluid/platform/enforce.h | 11 ++++++++--- paddle/fluid/platform/macros.h | 2 ++ paddle/fluid/platform/port.h | 9 ++++----- python/paddle/fluid/core.py | 26 +++++++++++++++++--------- 8 files changed, 44 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b1554fba5e1fa..fa87cc14f2668 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,6 +131,7 @@ option(WITH_LITE "Compile Paddle Fluid with Lite Engine" OFF) option(WITH_NCCL "Compile PaddlePaddle with NCCL support" ON) option(WITH_CRYPTO "Compile PaddlePaddle with crypto support" ON) option(WITH_ARM "Compile PaddlePaddle with arm support" OFF) +option(WITH_MUSL "Compile with musl libc instead of gblic" OFF) # PY_VERSION if(NOT PY_VERSION) diff --git a/cmake/configure.cmake b/cmake/configure.cmake index cf458d9770675..fc984f5e560ef 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -51,6 +51,16 @@ if(WIN32) endif(NOT MSVC) endif(WIN32) +if(WITH_MUSL) + add_definitions(-DPADDLE_WITH_MUSL) + + message(STATUS, "Set compile option WITH_MKL=OFF when WITH_MUSL=ON") + SET(WITH_MKL OFF) + + message(STATUS, "Set compile option WITH_GPU=OFF when WITH_MUSL=ON") + SET(WITH_GPU OFF) +endif() + if(WITH_PSLIB) add_definitions(-DPADDLE_WITH_PSLIB) endif() diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index f85e1f6511656..6d35d3395ba60 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -88,7 +88,7 @@ if(NOT APPLE AND NOT WIN32) set_target_properties(paddle_fluid_shared PROPERTIES LINK_FLAGS "${LINK_FLAGS}") # check symbol hidden FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/check_symbol.cmake - "execute_process(COMMAND bash -c \"${CMAKE_CURRENT_SOURCE_DIR}/check_symbol.sh" + "execute_process(COMMAND sh -c \"${CMAKE_CURRENT_SOURCE_DIR}/check_symbol.sh" " ${CMAKE_CURRENT_BINARY_DIR}/libpaddle_fluid.so\" RESULT_VARIABLE symbol_res)\n" "if(NOT \"\${symbol_res}\" STREQUAL \"0\")\n" " message(FATAL_ERROR \"Check symbol failed.\")\n" diff --git a/paddle/fluid/inference/check_symbol.sh b/paddle/fluid/inference/check_symbol.sh index b6b7d1f20baf7..a0f64796576c8 100755 --- a/paddle/fluid/inference/check_symbol.sh +++ b/paddle/fluid/inference/check_symbol.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh lib=$1 if [ $# -ne 1 ]; then echo "No input library"; exit -1 ; fi diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index a3ae9e48eea30..165321d9c87ff 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -47,6 +47,10 @@ limitations under the License. */ #include #include +#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL) +#include +#endif + #define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h #include "glog/logging.h" #include "paddle/fluid/platform/errors.h" @@ -236,13 +240,14 @@ inline std::string SimplifyDemangleStr(std::string str) { } inline std::string GetCurrentTraceBackString() { - static constexpr int TRACE_STACK_LIMIT = 100; std::ostringstream sout; sout << "\n\n--------------------------------------\n"; sout << "C++ Traceback (most recent call last):"; sout << "\n--------------------------------------\n"; -#if !defined(_WIN32) +#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL) + static constexpr int TRACE_STACK_LIMIT = 100; + void* call_stack[TRACE_STACK_LIMIT]; auto size = backtrace(call_stack, TRACE_STACK_LIMIT); auto symbols = backtrace_symbols(call_stack, size); @@ -261,7 +266,7 @@ inline std::string GetCurrentTraceBackString() { } free(symbols); #else - sout << "Windows not support stack backtrace yet.\n"; + sout << "Not support stack backtrace yet.\n"; #endif return sout.str(); } diff --git a/paddle/fluid/platform/macros.h b/paddle/fluid/platform/macros.h index 32b7efc04c1f2..fb5cf9fb31915 100644 --- a/paddle/fluid/platform/macros.h +++ b/paddle/fluid/platform/macros.h @@ -25,6 +25,8 @@ limitations under the License. */ classname& operator=(classname&&) = delete #endif +#ifndef PADDLE_WITH_MUSL #if defined(__FLT_MAX__) #define FLT_MAX __FLT_MAX__ #endif // __FLT_MAX__ +#endif // PADDLE_WITH_MUSL diff --git a/paddle/fluid/platform/port.h b/paddle/fluid/platform/port.h index c1b81159aca97..c5e8ff807a2d3 100644 --- a/paddle/fluid/platform/port.h +++ b/paddle/fluid/platform/port.h @@ -14,19 +14,18 @@ #pragma once -#include -#include - #include + +#include #include +#include #include #define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h #include "glog/logging.h" #if !defined(_WIN32) -#include // dladdr -#include // backtrace +#include // dladdr #include #include #include // std::accumulate diff --git a/python/paddle/fluid/core.py b/python/paddle/fluid/core.py index 9a14c4cdf14a4..ad116c2597064 100644 --- a/python/paddle/fluid/core.py +++ b/python/paddle/fluid/core.py @@ -205,8 +205,15 @@ def pre_load(dso_name): load_dso(dso_path) -def get_glibc_ver(): - return run_shell_command("ldd --version | awk '/ldd/{print $NF}'") +def get_libc_ver(): + ldd_glibc = run_shell_command("ldd --version | awk '/ldd/{print $NF}'") + if ldd_glibc is not None: + return ("glibc", ldd_glibc) + + ldd_musl = run_shell_command("ldd 2>&1 | awk '/Version/{print $NF}'") + if ldd_musl is not None: + return ("musl", ldd_musl) + return (None, None) def less_than_ver(a, b): @@ -231,13 +238,14 @@ def to_list(s): # For paddle, the problem is that 'libgomp' is a DSO with static TLS, and it is loaded after 14 DSOs. # So, here is a tricky way to solve the problem by pre load 'libgomp' before 'core_avx.so'. # The final solution is to upgrade glibc to > 2.22 on the target system. -if platform.system().lower() == 'linux' and less_than_ver(get_glibc_ver(), - '2.23'): - try: - pre_load('libgomp') - except Exception as e: - # NOTE(zhiqiu): do not abort if failed, since it may success when import core_avx.so - sys.stderr.write('Error: Can not preload libgomp.so') +if platform.system().lower() == 'linux': + libc_type, libc_ver = get_libc_ver() + if libc_type == 'glibc' and less_than_ver(libc_ver, '2.23'): + try: + pre_load('libgomp') + except Exception as e: + # NOTE(zhiqiu): do not abort if failed, since it may success when import core_avx.so + sys.stderr.write('Error: Can not preload libgomp.so') load_noavx = False