From f2ff6b6ea9c06cfe3549b6063e5b37f6ceb45f85 Mon Sep 17 00:00:00 2001 From: EdAtkin Date: Wed, 4 Dec 2024 13:23:56 +0000 Subject: [PATCH 1/2] upading to have a seperate target for GPU compile options, this makes setting up NuOscillator much easier as we need to pass the CPU compiler flags to NuOscillator --- CMakeLists.txt | 33 +++++++++++++++++++++++---- cmake/Modules/CUDASamples.cmake | 2 +- cmake/Modules/CUDASetup.cmake | 10 ++++---- cmake/Modules/NuOscillatorSetup.cmake | 14 +++++++----- manager/CMakeLists.txt | 6 ++++- splines/CMakeLists.txt | 6 ++++- 6 files changed, 53 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd14b63d9..d5f052585 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,8 @@ if(CUDAToolkit_FOUND AND NOT(USE_CPU)) message(STATUS "CUDA found. Adding CUDA support.") set(MaCh3_GPU_ENABLED TRUE) set(CPU_ONLY FALSE) + #Create a seperate target for GPU compiler options + add_library(MaCh3GPUCompilerOptions INTERFACE) else() message(STATUS "CUDA not found. Proceeding without CUDA support.") set(MaCh3_GPU_ENABLED FALSE) @@ -154,9 +156,9 @@ if(CPU_ONLY) target_compile_definitions(MaCh3CompilerOptions INTERFACE CPU_ONLY) endif() -if(MaCh3_GPU_ENABLED) - target_compile_definitions(MaCh3CompilerOptions INTERFACE CUDA) - target_compile_definitions(MaCh3CompilerOptions INTERFACE GPU_ON) +if(MaCh3_GPU_ENABLED) + target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE CUDA) + target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE GPU_ON) endif() if(NOT DEFINED MaCh3_LOW_MEMORY_STRUCTS_ENABLED) @@ -165,12 +167,22 @@ endif() if(MaCh3_LOW_MEMORY_STRUCTS_ENABLED) target_compile_definitions(MaCh3CompilerOptions INTERFACE _LOW_MEMORY_STRUCTS_) + # ETA - think the _LOW_MEMORY_STRUCTS_ also needs this definition + target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE _LOW_MEMORY_STRUCTS_) endif() set_target_properties(MaCh3CompilerOptions PROPERTIES EXPORT_NAME CompilerOptions) -install(TARGETS MaCh3CompilerOptions +if(MaCh3_GPU_ENABLED) + set_target_properties(MaCh3GPUCompilerOptions PROPERTIES EXPORT_NAME GPUCompilerOptions) + #If GPU enable then install both the CPU and GPU options + install(TARGETS MaCh3CompilerOptions MaCh3GPUCompilerOptions + EXPORT MaCh3-targets + LIBRARY DESTINATION lib/) +else() + install(TARGETS MaCh3CompilerOptions EXPORT MaCh3-targets LIBRARY DESTINATION lib/) +endif() #Include logger include(${CMAKE_CURRENT_LIST_DIR}/cmake/Modules/Logger.cmake) @@ -185,6 +197,15 @@ get_target_property(link_libraries MaCh3CompilerOptions INTERFACE_LINK_LIBRARIES cmessage(STATUS "Compile options for MaCh3CompilerOptions: ${compile_options}") cmessage(STATUS "Link libraries for MaCh3CompilerOptions: ${link_libraries}") +if(MaCh3_GPU_ENABLED) + #KS: Retrieve and print the compile options and link libraries for maximal verbose for GPU + get_target_property(gpu_compile_options MaCh3GPUCompilerOptions INTERFACE_COMPILE_OPTIONS) + get_target_property(gpu_link_libraries MaCh3GPUCompilerOptions INTERFACE_LINK_LIBRARIES) + #Nice print out of the GPU options if GPU is enabled + cmessage(STATUS "Compile options for MaCh3GPUCompilerOptions: ${gpu_compile_options}") + cmessage(STATUS "Link libraries for MaCh3GPUCompilerOptions: ${gpu_link_libraries}") +endif() + LIST(APPEND ALL_FEATURES DEBUG MULTITHREAD @@ -219,6 +240,10 @@ endif() #Anything that links to "MaCh3" will get all of these target properties add_library(MaCh3 INTERFACE) target_link_libraries(MaCh3 INTERFACE MCMC SamplePDF Covariance Splines Manager MaCh3CompilerOptions Plotting) +if(MaCh3_GPU_ENABLED) + #If GPU is enabled also link this to the MaCh3 target + target_link_libraries(MaCh3 INTERFACE Splines MaCh3GPUCompilerOptions) +endif() set_target_properties(MaCh3 PROPERTIES EXPORT_NAME All) install(TARGETS MaCh3 diff --git a/cmake/Modules/CUDASamples.cmake b/cmake/Modules/CUDASamples.cmake index d25772339..8a1843936 100644 --- a/cmake/Modules/CUDASamples.cmake +++ b/cmake/Modules/CUDASamples.cmake @@ -30,7 +30,7 @@ if(NOT CUDASAMPLES_FOUND) endif() cmessage(STATUS "Using the following CUDA samples paths: ${CMAKE_CUDA_SAMPLES_PATH}") -target_include_directories(MaCh3CompilerOptions INTERFACE +target_include_directories(MaCh3GPUCompilerOptions INTERFACE "$" "$" ) diff --git a/cmake/Modules/CUDASetup.cmake b/cmake/Modules/CUDASetup.cmake index 7fe7ad7f5..b8648a46a 100644 --- a/cmake/Modules/CUDASetup.cmake +++ b/cmake/Modules/CUDASetup.cmake @@ -47,7 +47,7 @@ string(REPLACE ";" " " CUDA_ARCHITECTURES_STR "${CMAKE_CUDA_ARCHITECTURES}") cmessage(STATUS "Using following CUDA architectures: ${CUDA_ARCHITECTURES_STR}") if(NOT MaCh3_DEBUG_ENABLED) - target_compile_options(MaCh3CompilerOptions INTERFACE + target_compile_options(MaCh3GPUCompilerOptions INTERFACE "$<$:-prec-sqrt=false;-use_fast_math;-O3;-Werror;cross-execution-space-call;-w>" "$<$:-Xptxas=-allow-expensive-optimizations=true;-Xptxas=-fmad=true;-Xptxas=-O3;>" "$<$:-Xcompiler=-fpic;-Xcompiler=-O3;-Xcompiler=-Wall;-Xcompiler=-Wextra;-Xcompiler=-Werror;-Xcompiler=-Wno-error=unused-parameter>" @@ -55,15 +55,15 @@ if(NOT MaCh3_DEBUG_ENABLED) else() #CW: -g and -G for debug flags to use cuda-gdb; slows stuff A LOT #-pxtas-options=-v, -maxregcount=N - target_compile_options(MaCh3CompilerOptions INTERFACE + target_compile_options(MaCh3GPUCompilerOptions INTERFACE "$<$:-prec-sqrt=false;-use_fast_math;-Werror;cross-execution-space-call;-w>" "$<$:-Xcompiler=-g;>" "$<$:-Xptxas=-dlcm=ca;-Xptxas=-warn-lmem-usage;-Xptxas=-warn-spills;-Xptxas=-v;-Xcompiler=-Wall;-Xcompiler=-Wextra;-Xcompiler=-Werror;-Xcompiler=-Wno-error=unused-parameter>" ) - target_compile_definitions(MaCh3CompilerOptions INTERFACE "$<$:CUDA_ERROR_CHECK>") + target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE "$<$:CUDA_ERROR_CHECK>") endif() -target_include_directories(MaCh3CompilerOptions INTERFACE ${CUDAToolkit_INCLUDE_DIRS}) +target_include_directories(MaCh3GPUCompilerOptions INTERFACE ${CUDAToolkit_INCLUDE_DIRS}) if(MaCh3_DEBUG_ENABLED) include(${CMAKE_CURRENT_LIST_DIR}/CUDASamples.cmake) @@ -76,7 +76,7 @@ if(NOT DEFINED NSplines_GPU) endif() # Pass NSplines_GPU as a preprocessor definition to the compiler -target_compile_definitions(MaCh3CompilerOptions INTERFACE NSplines_GPU=${NSplines_GPU}) +target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE NSplines_GPU=${NSplines_GPU}) cmessage(STATUS "Using \"${NSplines_GPU}\" for GPU EventByEvent Splines") diff --git a/cmake/Modules/NuOscillatorSetup.cmake b/cmake/Modules/NuOscillatorSetup.cmake index 6417814f0..c4ba1462a 100644 --- a/cmake/Modules/NuOscillatorSetup.cmake +++ b/cmake/Modules/NuOscillatorSetup.cmake @@ -47,20 +47,22 @@ IsTrue(MaCh3_MULTITHREAD_ENABLED DAN_USE_MULTITHREAD) IsTrue(MaCh3_LOW_MEMORY_STRUCTS_ENABLED DAN_DOUBLE) SwitchLogic(DAN_DOUBLE) -# Get the compile options for MaCh3CompilerOptions -get_target_property(compile_options MaCh3CompilerOptions INTERFACE_COMPILE_OPTIONS) +# Get the CPU compile options for MaCh3CompilerOptions +get_target_property(cpu_compile_options MaCh3CompilerOptions INTERFACE_COMPILE_OPTIONS) # Join the compile options list into a space-separated string -string(REPLACE ";" " " compile_options_string "${compile_options}") +string(REPLACE ";" " " compile_options_string "${cpu_compile_options}") + #KS: This may seem hacky, but when CMAKE_CUDA_ARCHITECTURES is passed, it's treated as a string rather than a list. Since CMake uses semi-colon-delimited strings to represent lists, we convert it to a proper list to handle CUDA architectures correctly. set(CMAKE_CUDA_ARCHITECTURES_STRING ${CMAKE_CUDA_ARCHITECTURES}) -string(REPLACE " " ";" CMAKE_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}") +#string(REPLACE " " ";" CMAKE_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}") +string(REPLACE " " ";" CMAKE_CUDA_ARCHITECTURES_STRING "${CMAKE_CUDA_ARCHITECTURES}") #Try adding Oscillator Class CPMAddPackage( NAME NuOscillator - VERSION 1.0.0 + VERSION 1.0.3 GITHUB_REPOSITORY "dbarrow257/NuOscillator" GIT_TAG "tags/v1.0.0" OPTIONS @@ -74,7 +76,7 @@ CPMAddPackage( "UseProb3ppLinear ${USE_Prob3ppLinear}" "UseNuFASTLinear ${USE_NuFastLiner}" - "NuOscillator_Compiler_Flags ${compile_options_string}" + "NuOscillator_Compiler_Flags ${cpu_compile_options}" "CMAKE_CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES_STRING}" ) diff --git a/manager/CMakeLists.txt b/manager/CMakeLists.txt index b612c3dec..58fe7445f 100644 --- a/manager/CMakeLists.txt +++ b/manager/CMakeLists.txt @@ -23,9 +23,13 @@ if(NOT CPU_ONLY) set_target_properties(Manager PROPERTIES CUDA_SEPARABLE_COMPILATION ON) #KS: In cmake 3.18 this is no longer needed https://cmake.org/cmake/help/latest/variable/CMAKE_CUDA_ARCHITECTURES.html#cmake-cuda-architectures set_property(TARGET Manager PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}) + #If compiling with GPU link both CPU and GPU compiler options + target_link_libraries(Manager yaml-cpp spdlog ROOT::ROOT MaCh3CompilerOptions MaCh3GPUCompilerOptions) +else() + target_link_libraries(Manager yaml-cpp spdlog ROOT::ROOT MaCh3CompilerOptions) endif() -target_link_libraries(Manager yaml-cpp spdlog ROOT::ROOT MaCh3CompilerOptions) +#target_link_libraries(Manager yaml-cpp spdlog ROOT::ROOT MaCh3CompilerOptions) target_include_directories(Manager PUBLIC $ $) diff --git a/splines/CMakeLists.txt b/splines/CMakeLists.txt index 65c5c2e81..cfb5205a6 100644 --- a/splines/CMakeLists.txt +++ b/splines/CMakeLists.txt @@ -29,7 +29,11 @@ set_target_properties(Splines PROPERTIES PUBLIC_HEADER "${HEADERS}" EXPORT_NAME Splines) -target_link_libraries(Splines Manager MaCh3CompilerOptions) +if(NOT CPU_ONLY) + target_link_libraries(Splines Manager MaCh3CompilerOptions MaCh3GPUCompilerOptions) +else() + target_link_libraries(Splines Manager MaCh3CompilerOptions) +endif() target_include_directories(Splines PUBLIC $ From 38a18878578e533550bc68f22027d883a78d902b Mon Sep 17 00:00:00 2001 From: EdAtkin Date: Tue, 10 Dec 2024 13:10:10 +0000 Subject: [PATCH 2/2] Updated so that if GPU is not enabled the GPU compiler options are just empty, this means fewer if statements. Also removed the GPU setting for LOW_MEMORY_STUCTS as this isn't used --- CMakeLists.txt | 27 +++++++-------------------- manager/CMakeLists.txt | 8 +++----- splines/CMakeLists.txt | 7 +------ 3 files changed, 11 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b374fc83..f1c7ce0f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,10 @@ option(MaCh3_WERROR_ENABLED "Whether to build MaCh3 with heightened compiler ped # Try to find CUDA add_library(MaCh3CompileDefinitions INTERFACE) +#Create a seperate target for GPU compiler options +#if GPU is not enabled nothing will be added to this and it will be empty +# this saves a few if statements +add_library(MaCh3GPUCompilerOptions INTERFACE) find_package(CUDAToolkit QUIET) # Check if CUDA was found @@ -21,8 +25,6 @@ if(CUDAToolkit_FOUND AND NOT(USE_CPU)) set(MaCh3_GPU_ENABLED TRUE) enable_language(CUDA) set(CPU_ONLY FALSE) - #Create a seperate target for GPU compiler options - add_library(MaCh3GPUCompilerOptions INTERFACE) else() message(STATUS "CUDA not found. Proceeding without CUDA support.") set(MaCh3_GPU_ENABLED FALSE) @@ -181,24 +183,13 @@ endif() if(MaCh3_LOW_MEMORY_STRUCTS_ENABLED) target_compile_definitions(MaCh3CompilerOptions INTERFACE _LOW_MEMORY_STRUCTS_) - if(MaCh3_GPU_ENABLED) - # ETA - think the _LOW_MEMORY_STRUCTS_ also needs this definition - target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE _LOW_MEMORY_STRUCTS_) - endif(MaCh3_GPU_ENABLED) endif(MaCh3_LOW_MEMORY_STRUCTS_ENABLED) set_target_properties(MaCh3CompilerOptions PROPERTIES EXPORT_NAME CompilerOptions) -if(MaCh3_GPU_ENABLED) - set_target_properties(MaCh3GPUCompilerOptions PROPERTIES EXPORT_NAME GPUCompilerOptions) - #If GPU enable then install both the CPU and GPU options - install(TARGETS MaCh3CompilerOptions MaCh3GPUCompilerOptions MaCh3CompileDefinitions - EXPORT MaCh3-targets - LIBRARY DESTINATION lib/) -else() - install(TARGETS MaCh3CompilerOptions MaCh3CompileDefinitions +set_target_properties(MaCh3GPUCompilerOptions PROPERTIES EXPORT_NAME GPUCompilerOptions) +install(TARGETS MaCh3CompilerOptions MaCh3GPUCompilerOptions MaCh3CompileDefinitions EXPORT MaCh3-targets LIBRARY DESTINATION lib/) -endif() #Include logger include(${CMAKE_CURRENT_LIST_DIR}/cmake/Modules/Logger.cmake) @@ -254,11 +245,7 @@ endif() #This is to export the target properties of MaCh3 #Anything that links to "MaCh3" will get all of these target properties add_library(MaCh3 INTERFACE) -target_link_libraries(MaCh3 INTERFACE MCMC SamplePDF Covariance Splines Manager MaCh3CompilerOptions Plotting) -if(MaCh3_GPU_ENABLED) - #If GPU is enabled also link this to the MaCh3 target - target_link_libraries(MaCh3 INTERFACE Splines MaCh3GPUCompilerOptions) -endif() +target_link_libraries(MaCh3 INTERFACE MCMC SamplePDF Covariance Splines Manager MaCh3CompilerOptions MaCh3GPUCompilerOptions Plotting) set_target_properties(MaCh3 PROPERTIES EXPORT_NAME All) install(TARGETS MaCh3 diff --git a/manager/CMakeLists.txt b/manager/CMakeLists.txt index ccca0bd61..c0bd1ab3e 100644 --- a/manager/CMakeLists.txt +++ b/manager/CMakeLists.txt @@ -22,13 +22,11 @@ set_target_properties(Manager PROPERTIES if(NOT CPU_ONLY) set_target_properties(Manager PROPERTIES CUDA_SEPARABLE_COMPILATION ON) #KS: In cmake 3.18 this is no longer needed https://cmake.org/cmake/help/latest/variable/CMAKE_CUDA_ARCHITECTURES.html#cmake-cuda-architectures - set_property(TARGET Manager PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}) - #If compiling with GPU link both CPU and GPU compiler options - target_link_libraries(Manager PUBLIC MaCh3CompilerOptions MaCh3GPUCompilerOptions yaml-cpp spdlog ROOT::Tree ROOT::Hist) -else() - target_link_libraries(Manager PUBLIC MaCh3CompilerOptions yaml-cpp spdlog ROOT::Tree ROOT::Hist) + set_property(TARGET Manager PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}) endif() +#If compiling with GPU is not enabled MaCh3GPUCompilerOptions will be empty +target_link_libraries(Manager PUBLIC MaCh3CompilerOptions MaCh3GPUCompilerOptions yaml-cpp spdlog ROOT::Tree ROOT::Hist) target_link_libraries(Manager PRIVATE MaCh3Warnings) target_include_directories(Manager PUBLIC diff --git a/splines/CMakeLists.txt b/splines/CMakeLists.txt index 407c40d08..34cace232 100644 --- a/splines/CMakeLists.txt +++ b/splines/CMakeLists.txt @@ -30,12 +30,7 @@ set_target_properties(Splines PROPERTIES PUBLIC_HEADER "${HEADERS}" EXPORT_NAME Splines) -if(NOT CPU_ONLY) - target_link_libraries(Splines PUBLIC Manager MaCh3CompilerOptions MaCh3GPUCompilerOptions) -else() - target_link_libraries(Splines PUBLIC Manager MaCh3CompilerOptions) -endif() - +target_link_libraries(Splines PUBLIC Manager MaCh3CompilerOptions MaCh3GPUCompilerOptions) target_link_libraries(Splines PRIVATE MaCh3Warnings) target_include_directories(Splines PUBLIC