diff --git a/CMakeLists.txt b/CMakeLists.txt index eb6afd8f..f0a29821 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ endif() set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH};") set(AMS_APP_LIBRARIES "") +set(AMS_APP_PRIVATE_LIBRARIES "") set(AMS_APP_DEFINES "") set(AMS_APP_INCLUDES "") @@ -39,6 +40,8 @@ option(WITH_TORCH_DEBUG "Compute RMSE of Surrogate Model and Physics Module" option(WITH_TESTS "Compile tests" OFF) option(WITH_REDIS "Use REDIS as a database back end" OFF) option(WITH_HDF5 "Use HDF5 as a database back end" OFF) +option(HDF5_USE_STATIC_LIBRARIES "Use static HDF5." OFF) +set(HDF5_WITH_ZLIB "" CACHE FILETYPE "Use the following zlib for HDF5") option(WITH_RMQ "Use RabbitMQ as a database back end (require a reachable and running RabbitMQ server service)" OFF) option(WITH_AMS_DEBUG "Enable verbose messages" OFF) option(WITH_PERFFLOWASPECT "Use PerfFlowAspect for Profiling" OFF) @@ -46,6 +49,7 @@ option(WITH_WORKFLOW "Install python drivers used by the outer workflow" O option(WITH_AMS_LIB "Install C++ library to support scientific applications" ON) option(WITH_ADIAK "Use Adiak for recording metadata" OFF) option(BUILD_SHARED_LIBS "Build using shared libraries" ON) +option(EXCLUDE_STATIC_LIBS "Exclude static libs from the linking line" OFF) if (WITH_MPI) # SET(CMAKE_CXX_COMPILER "${MPI_CXX_COMPILER}" CACHE FILEPATH "CXX compiler overridden with MPI C++ wrapper") @@ -56,7 +60,6 @@ if (WITH_MPI) message(STATUS "MPICXX: ${MPI_CXX_COMPILER}") list(APPEND AMS_APP_LIBRARIES MPI::MPI_CXX) message(STATUS "MPI Library used: " MPI::MPI_CXX) - list(APPEND AMS_APP_DEFINES "-D__ENABLE_MPI__") endif() # ------------------------------------------------------------------------------ @@ -69,6 +72,7 @@ if (WITH_CUDA) if (BUILD_SHARED_LIBS) set(CUDA_RUNTIME_LIBRARY "Shared") else() + set(HDF5_USE_STATIC_LIBRARIES ON) set(CUDA_RUNTIME_LIBRARY "Static") endif() @@ -132,14 +136,17 @@ endif() # WITH_REDIS if (WITH_HDF5) if (HDF5_USE_STATIC_LIBRARIES) - find_package(HDF5 NAMES hdf5 COMPONENTS C static NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake) - list(APPEND AMS_APP_LIBRARIES ${HDF5_C_STATIC_LIBRARY}) - message(STATUS "HDF5 Static Library : ${HDF5_C_STATIC_LIBRARY}") -else() - find_package(HDF5 NAMES hdf5 COMPONENTS C shared NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake) - list(APPEND AMS_APP_LIBRARIES ${HDF5_C_SHARED_LIBRARY}) - message(STATUS "HDF5 Shared Library : ${HDF5_C_SHARED_LIBRARY}") -endif() + find_package(HDF5 NAMES hdf5 COMPONENTS C static NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake) + list(APPEND AMS_APP_LIBRARIES ${HDF5_C_STATIC_LIBRARY}) + message(STATUS "HDF5 Static Library : ${HDF5_C_STATIC_LIBRARY}") + else() + find_package(HDF5 NAMES hdf5 COMPONENTS C shared NO_DEFAULT_PATH PATHS ${AMS_HDF5_DIR} ${AMS_HDF5_DIR}/share/cmake) + list(APPEND AMS_APP_LIBRARIES ${HDF5_C_SHARED_LIBRARY}) + message(STATUS "HDF5 Shared Library : ${HDF5_C_SHARED_LIBRARY}") + endif() + if (NOT HDF5_WITH_ZLIB STREQUAL "") + list(APPEND AMS_APP_LIBRARIES ${HDF5_WITH_ZLIB}) + endif() list(APPEND AMS_APP_INCLUDES ${HDF5_INCLUDE_DIR}) list(APPEND AMS_APP_DEFINES "-D__ENABLE_HDF5__") message(STATUS "HDF5 Include directories: ${HDF5_INCLUDE_DIR}") @@ -184,7 +191,7 @@ list(APPEND AMS_APP_LIBRARIES umpire) list(APPEND AMS_APP_INCLUDES ${UMPIRE_INCLUDE_DIR}) find_package(nlohmann_json REQUIRED) -list(APPEND AMS_APP_LIBRARIES nlohmann_json::nlohmann_json) +list(APPEND AMS_APP_PRIVATE_LIBRARIES nlohmann_json::nlohmann_json) # ------------------------------------------------------------------------------ find_package(Threads REQUIRED) @@ -194,16 +201,50 @@ if (WITH_TORCH) find_package(Torch REQUIRED) # This is annoying, torch populates all my cuda flags # and resets them - set(CMAKE_CUDA_FLAGS "") + # set(CMAKE_CUDA_FLAGS "") set(CMAKE_CUDA_ARCHITECTURES ON) - list(APPEND AMS_APP_INCLUDES "${TORCH_INCLUDE_DIRS}") - list(APPEND AMS_APP_LIBRARIES "${TORCH_LIBRARIES}") - + #get_target_property(torch_interface_system_includes + # torch INTERFACE_SYSTEM_INCLUDE_DIRECTORIES) + #if ( torch_interface_system_includes ) + # list(APPEND AMS_APP_INCLUDES ${torch_interface_system_includes}) + #endif() + # + #get_target_property(torch_interface_includes + # torch INTERFACE_INCLUDE_DIRECTORIES) + #if ( torch_interface_includes ) + # list(APPEND AMS_APP_INCLUDES ${torch_interface_includes}) + #endif() + # + #get_target_property(torch_interface_defines + # torch INTERFACE_COMPILE_DEFINITIONS) + #if ( troch_interface_defines ) + # list(APPEND AMS_APP_DEFINES ${torch_interface_defines}) + #endif() + # + #get_target_property(torch_interface_compile_options + # torch INTERFACE_COMPILE_OPTIONS) + #if ( torch_interface_compile_options ) + # list(APPEND AMS_APP_DEFINES ${torch_interface_compile_options}) + #endif() + # + #get_target_property(_interface_link_directories + # ${arg_FROM} INTERFACE_LINK_DIRECTORIES) + #if ( _interface_link_directories ) + # target_link_directories( ${arg_TO} ${_scope} ${_interface_link_directories}) + #endif() + # + #get_target_property(torch_interface_link_libraries + # torch INTERFACE_LINK_LIBRARIES) + #if ( torch_interface_link_libraries ) + # list(APPEND AMS_APP_LIBRARIES "${torch_interface_link_libraries}") + #endif() + + list(APPEND AMS_TORCH_LIBRARY torch) list(APPEND AMS_APP_DEFINES "-D__ENABLE_TORCH__") - set(BLA_VENDER OpenBLAS) - find_package(BLAS REQUIRED) - list(APPEND AMS_APP_LIBRARIES "${BLAS_LIBRARIES}") + #set(BLA_VENDER OpenBLAS) + #find_package(BLAS REQUIRED) + #list(APPEND AMS_APP_LIBRARIES "${BLAS_LIBRARIES}") endif() # ------------------------------------------------------------------------------ @@ -256,6 +297,98 @@ if (WITH_PERFFLOWASPECT) endif() +macro(inherit_target_nostatic) + set(options) + set(singleValueArgs TO FROM OBJECT) + set(multiValueArgs) + + # Parse the arguments + cmake_parse_arguments(arg "${options}" "${singleValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + # Check arguments + if ( NOT DEFINED arg_TO ) + message( FATAL_ERROR "Must provide a TO argument to the 'blt_inherit_target' macro" ) + endif() + + if ( NOT DEFINED arg_FROM ) + message( FATAL_ERROR "Must provide a FROM argument to the 'blt_inherit_target' macro" ) + endif() + + set(_scope INTERFACE) + + get_target_property(_interface_system_includes + ${arg_FROM} INTERFACE_SYSTEM_INCLUDE_DIRECTORIES) + if ( _interface_system_includes ) + target_include_directories(${arg_TO} SYSTEM ${_scope} ${_interface_system_includes}) + endif() + + get_target_property(_interface_includes + ${arg_FROM} INTERFACE_INCLUDE_DIRECTORIES) + if ( _interface_includes ) + target_include_directories(${arg_TO} ${_scope} ${_interface_includes}) + endif() + + get_target_property(_interface_defines + ${arg_FROM} INTERFACE_COMPILE_DEFINITIONS) + if ( _interface_defines ) + target_compile_definitions( ${arg_TO} ${_scope} ${_interface_defines}) + endif() + + if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13.0" ) + get_target_property(_interface_link_options + ${arg_FROM} INTERFACE_LINK_OPTIONS) + if ( _interface_link_options ) + target_link_options( ${arg_TO} ${_scope} ${_interface_link_options}) + endif() + endif() + + get_target_property(_interface_compile_options + ${arg_FROM} INTERFACE_COMPILE_OPTIONS) + if ( _interface_compile_options ) + target_compile_options( ${arg_TO} ${_scope} ${_interface_compile_options}) + endif() + + if ( NOT arg_OBJECT ) + #get_target_property(_interface_link_directories + # ${arg_FROM} INTERFACE_LINK_DIRECTORIES) + #if ( _interface_link_directories ) + # target_link_directories( ${arg_TO} ${_scope} ${_interface_link_directories}) + #endif() + + #get_target_property(_interface_link_libraries + # ${arg_FROM} INTERFACE_LINK_LIBRARIES) + #if ( _interface_link_libraries ) + # target_link_libraries( ${arg_TO} ${_scope} ${_interface_link_libraries}) + #endif() + endif() + +endmacro(inherit_target_nostatic) + + +if (EXCLUDE_STATIC_LIBS) + set(NONSTATIC_AMS_APP_INTERFACE_LIBRARIES "") + set(NONSTATIC_AMS_APP_PRIVATE_LIBRARIES "") + foreach (THIS_LIB ${AMS_APP_LIBRARIES}) + list(APPEND NONSTATIC_AMS_APP_INTERFACE_LIBRARIES ${THIS_LIB}) + if (TARGET ${THIS_LIB}) + get_target_property(target_type ${THIS_LIB} TYPE) + if (target_type STREQUAL STATIC_LIBRARY) + add_library("${THIS_LIB}::nostatic" INTERFACE IMPORTED) + inherit_target_nostatic(TO "${THIS_LIB}::nostatic" FROM ${THIS_LIB}) + list(APPEND NONSTATIC_AMS_APP_PRIVATE_LIBRARIES "${THIS_LIB}::nostatic") + else() + list(APPEND NONSTATIC_AMS_APP_PRIVATE_LIBRARIES ${THIS_LIB}) + endif() + else() + get_filename_component(THIS_EXT ${THIS_LIB} EXT) + if (NOT ".a" STREQUAL "${THIS_EXT}") + list(APPEND NONSTATIC_AMS_APP_PRIVATE_LIBRARIES ${THIS_LIB}) + endif() + endif() + endforeach() +endif() + add_subdirectory(src) # ------------------------------------------------------------------------------ diff --git a/examples/main.cpp b/examples/main.cpp index b0ea1eb5..2f34f5bc 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -219,7 +219,7 @@ int run(const char *device_name, std::default_random_engine generator; std::normal_distribution distribution(avg, stdDev); threshold = distribution(generator); -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ if (wS > 1) { if (rId == 0) { for (int i = 1; i < wS; i++) { diff --git a/src/AMSlib/AMS.cpp b/src/AMSlib/AMS.cpp index 5c5d3066..90d7811f 100644 --- a/src/AMSlib/AMS.cpp +++ b/src/AMSlib/AMS.cpp @@ -595,7 +595,7 @@ AMSExecutor AMSCreateExecutor(AMSCAbstrModel model, } } -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ AMSExecutor AMSCreateDistributedExecutor(AMSCAbstrModel model, AMSDType data_type, AMSResourceType resource_type, diff --git a/src/AMSlib/CMakeLists.txt b/src/AMSlib/CMakeLists.txt index 35159b0c..0ed740e5 100644 --- a/src/AMSlib/CMakeLists.txt +++ b/src/AMSlib/CMakeLists.txt @@ -50,7 +50,14 @@ target_include_directories(AMS PUBLIC $) target_include_directories(AMS PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_directories(AMS PUBLIC ${AMS_APP_LIB_DIRS}) -target_link_libraries(AMS PUBLIC ${AMS_APP_LIBRARIES} stdc++fs) +if (EXCLUDE_STATIC_LIBS) + target_link_libraries(AMS PRIVATE ${NONSTATIC_AMS_APP_PRIVATE_LIBRARIES} ${AMS_APP_PRIVATE_LIBRARIES} ${AMS_TORCH_LIBRARY} stdc++fs stdc++ /usr/tce/packages/intel/intel-2022.1.0/compiler/2022.1.0/linux/compiler/lib/intel64_lin/libintlc.so m) + target_link_libraries(AMS INTERFACE ${NONSTATIC_AMS_APP_INTERFACE_LIBRARIES} stdc++fs stdc++ /usr/tce/packages/intel/intel-2022.1.0/compiler/2022.1.0/linux/compiler/lib/intel64_lin/libintlc.so m) + target_link_options(AMS PRIVATE "-Wl,--unresolved-symbols=ignore-all") +else() + target_link_libraries(AMS PUBLIC ${AMS_APP_LIBRARIES} stdc++fs stdc++ /usr/tce/packages/intel/intel-2022.1.0/compiler/2022.1.0/linux/compiler/lib/intel64_lin/libintlc.so m) + target_link_libraries(AMS PRIVATE ${AMS_APP_PRIVATE_LIBRARIES}) +endif() #------------------------------------------------------------------------------- # create the configuration header file with the respective information @@ -83,7 +90,7 @@ install(TARGETS AMS DESTINATION lib) install(EXPORT AMSTargets - FILE AMS.cmake + FILE AMSConfig.cmake DESTINATION lib/cmake/AMS) install(FILES ${PROJECT_BINARY_DIR}/include/AMS.h DESTINATION include) diff --git a/src/AMSlib/include/AMS.h b/src/AMSlib/include/AMS.h index 719fe253..652ef9d9 100644 --- a/src/AMSlib/include/AMS.h +++ b/src/AMSlib/include/AMS.h @@ -20,14 +20,13 @@ #define CALIPER(stmt) #endif -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ #include #define MPI_CALL(stmt) \ if (stmt != MPI_SUCCESS) { \ fprintf(stderr, "Error in MPI-Call (File: %s, %d)\n", __FILE__, __LINE__); \ } #else -typedef void *MPI_Comm; #define MPI_CALL(stm) #endif @@ -78,7 +77,7 @@ AMSExecutor AMSCreateExecutor(AMSCAbstrModel model, int process_id, int world_size); -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ AMSExecutor AMSCreateDistributedExecutor(AMSCAbstrModel model, AMSDType data_type, AMSResourceType resource_type, diff --git a/src/AMSlib/wf/basedb.hpp b/src/AMSlib/wf/basedb.hpp index d6acc086..582ef799 100644 --- a/src/AMSlib/wf/basedb.hpp +++ b/src/AMSlib/wf/basedb.hpp @@ -1029,7 +1029,7 @@ class AMSMessage _data(nullptr), _total_size(0) { -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank)); #endif AMSMsgHeader header(_rank, @@ -1072,7 +1072,7 @@ class AMSMessage auto header = AMSMsgHeader::decode(data); int current_rank = 0; -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, ¤t_rank)); #endif _rank = header.mpi_rank; @@ -1234,7 +1234,7 @@ class RMQConsumerHandler final : public AMQP::LibEventHandler _messages(std::make_shared>()), _channel(nullptr) { -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank)); #endif } @@ -1473,7 +1473,7 @@ class RMQConsumer std::string queue) : _rank(0), _queue(queue), _cacert(cacert), _handler(nullptr) { -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank)); #endif #ifdef EVTHREAD_USE_PTHREADS_IMPLEMENTED @@ -1618,7 +1618,7 @@ class RMQPublisherHandler final : public AMQP::LibEventHandler _channel(nullptr), _rchannel(nullptr) { -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank)); #endif established = establish_connection.get_future(); @@ -2023,7 +2023,7 @@ class RMQPublisher _handler(nullptr), _buffer_msg(std::move(msgs_to_send)) { -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &_rank)); #endif #ifdef EVTHREAD_USE_PTHREADS_IMPLEMENTED diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index 9807461c..730e3655 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -25,7 +25,7 @@ #include "resource_manager.hpp" #include "wf/basedb.hpp" -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ #include #include "wf/redist_load.hpp" #endif @@ -80,7 +80,7 @@ class AMSWorkflow /** @brief execution policy of the distributed system. Load balance or not. */ AMSExecPolicy ePolicy; -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ /** @brief MPI Communicator for all ranks that call collectively the evaluate function **/ MPI_Comm comm; #endif @@ -169,7 +169,7 @@ class AMSWorkflow : AppCall(nullptr), DB(nullptr), appDataLoc(AMSResourceType::AMS_HOST), -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ comm(MPI_COMM_NULL), #endif ePolicy(AMSExecPolicy::AMS_UBALANCED) @@ -192,7 +192,7 @@ class AMSWorkflow wSize(_wSize), appDataLoc(app_data_loc), uqPolicy(uq_policy), -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ comm(MPI_COMM_NULL), #endif ePolicy(AMSExecPolicy::AMS_UBALANCED) @@ -208,7 +208,7 @@ class AMSWorkflow void set_physics(AMSPhysicFn _AppCall) { AppCall = _AppCall; } -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ void set_communicator(MPI_Comm communicator) { comm = communicator; } #endif @@ -216,7 +216,7 @@ class AMSWorkflow bool should_load_balance() const { -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ return (comm != MPI_COMM_NULL && ePolicy == AMSExecPolicy::AMS_BALANCED); #else return false; @@ -371,7 +371,7 @@ class AMSWorkflow // Simple modification can make it easier to read. // if (should_load_balance) -> Code for load balancing // else -> current code -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) AMSLoadBalancer lBalancer(rId, wSize, packedElements, comm); if (should_load_balance()) { @@ -391,7 +391,7 @@ class AMSWorkflow CALIPER(CALI_MARK_END("PHYSICS MODULE");) } -#ifdef __ENABLE_MPI__ +#ifdef __AMS_ENABLE_MPI__ CALIPER(CALI_MARK_BEGIN("LOAD BALANCE MODULE");) if (should_load_balance()) { lBalancer.gatherOutputs(packedOutputs, appDataLoc); diff --git a/tests/AMSlib/CMakeLists.txt b/tests/AMSlib/CMakeLists.txt index 5571a922..bead4393 100644 --- a/tests/AMSlib/CMakeLists.txt +++ b/tests/AMSlib/CMakeLists.txt @@ -132,9 +132,9 @@ endfunction() function(BUILD_TEST exe source) add_executable(${exe} ${source}) - target_include_directories(${exe} PRIVATE "${PROJECT_SOURCE_DIR}/src/AMSlib/" umpire ${caliper_INCLUDE_DIR} ${MPI_INCLUDE_PATH}) + target_include_directories(${exe} PRIVATE "${PROJECT_SOURCE_DIR}/src/AMSlib/" umpire ${AMS_APP_INCLUDES} ${caliper_INCLUDE_DIR} ${MPI_INCLUDE_PATH}) target_link_directories(${exe} PRIVATE ${AMS_APP_LIB_DIRS}) - target_link_libraries(${exe} PRIVATE AMS ${AMS_APP_LIBRARIES}) + target_link_libraries(${exe} PRIVATE AMS ${AMS_APP_LIBRARIES} ${AMS_TORCH_LIBRARY}) target_compile_definitions(${exe} PRIVATE ${AMS_APP_DEFINES})