Merge pull request #237 from mach3-software/feature_gpu_compile_options

Feature gpu compile options

CMakeLists.txt

@@ -13,6 +13,10 @@ option(MaCh3_WERROR_ENABLED "Whether to build MaCh3 with heightened compiler ped
 
 # Try to find CUDA
 add_library(MaCh3CompileDefinitions INTERFACE)
+#Create a seperate target for GPU compiler options
+#if GPU is not enabled nothing will be added to this and it will be empty
+# this saves a few if statements
+add_library(MaCh3GPUCompilerOptions INTERFACE)
 
 find_package(CUDAToolkit QUIET)
 # Check if CUDA was found
@@ -168,21 +172,22 @@ if(CPU_ONLY)
   target_compile_definitions(MaCh3CompileDefinitions INTERFACE CPU_ONLY)
 endif()
 
-if(MaCh3_GPU_ENABLED)
-  target_compile_definitions(MaCh3CompileDefinitions INTERFACE CUDA)
-  target_compile_definitions(MaCh3CompileDefinitions INTERFACE GPU_ON)
+if(MaCh3_GPU_ENABLED) 
+  target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE CUDA)
+  target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE GPU_ON)
 endif()
 
 if(NOT DEFINED MaCh3_LOW_MEMORY_STRUCTS_ENABLED)
   set(MaCh3_LOW_MEMORY_STRUCTS_ENABLED FALSE)
 endif()
 
 if(MaCh3_LOW_MEMORY_STRUCTS_ENABLED)
-  target_compile_definitions(MaCh3CompileDefinitions INTERFACE _LOW_MEMORY_STRUCTS_)
-endif()
+  target_compile_definitions(MaCh3CompilerOptions INTERFACE _LOW_MEMORY_STRUCTS_)
+endif(MaCh3_LOW_MEMORY_STRUCTS_ENABLED)
 
 set_target_properties(MaCh3CompilerOptions PROPERTIES EXPORT_NAME CompilerOptions)
-install(TARGETS MaCh3CompilerOptions MaCh3CompileDefinitions
+set_target_properties(MaCh3GPUCompilerOptions PROPERTIES EXPORT_NAME GPUCompilerOptions)
+install(TARGETS MaCh3CompilerOptions MaCh3GPUCompilerOptions MaCh3CompileDefinitions
     EXPORT MaCh3-targets
     LIBRARY DESTINATION lib/)
 
@@ -199,6 +204,15 @@ get_target_property(link_libraries MaCh3CompilerOptions INTERFACE_LINK_LIBRARIES
 cmessage(STATUS "Compile options for MaCh3CompilerOptions: ${compile_options}")
 cmessage(STATUS "Link libraries for MaCh3CompilerOptions: ${link_libraries}")
 
+if(MaCh3_GPU_ENABLED)
+  #KS: Retrieve and print the compile options and link libraries for maximal verbose for GPU
+  get_target_property(gpu_compile_options MaCh3GPUCompilerOptions INTERFACE_COMPILE_OPTIONS)
+  get_target_property(gpu_link_libraries MaCh3GPUCompilerOptions INTERFACE_LINK_LIBRARIES)
+  #Nice print out of the GPU options if GPU is enabled
+  cmessage(STATUS "Compile options for MaCh3GPUCompilerOptions: ${gpu_compile_options}")
+  cmessage(STATUS "Link libraries for MaCh3GPUCompilerOptions: ${gpu_link_libraries}")
+endif()
+
 LIST(APPEND ALL_FEATURES
   DEBUG
   MULTITHREAD
@@ -231,7 +245,7 @@ endif()
 #This is to export the target properties of MaCh3
 #Anything that links to "MaCh3" will get all of these target properties
 add_library(MaCh3 INTERFACE)
-target_link_libraries(MaCh3 INTERFACE MCMC SamplePDF Covariance Splines Manager MaCh3CompilerOptions Plotting)
+target_link_libraries(MaCh3 INTERFACE MCMC SamplePDF Covariance Splines Manager MaCh3CompilerOptions MaCh3GPUCompilerOptions Plotting)
 set_target_properties(MaCh3 PROPERTIES EXPORT_NAME All)
 
 install(TARGETS MaCh3

cmake/Modules/CUDASamples.cmake

@@ -30,7 +30,7 @@ if(NOT CUDASAMPLES_FOUND)
 endif()
 
 cmessage(STATUS "Using the following CUDA samples paths: ${CMAKE_CUDA_SAMPLES_PATH}")
-target_include_directories(MaCh3CompileDefinitions INTERFACE
+target_include_directories(MaCh3GPUCompilerOptions INTERFACE
     "$<BUILD_INTERFACE:${CMAKE_CUDA_SAMPLES_PATH}>"
     "$<INSTALL_INTERFACE:include>"
 )

cmake/Modules/CUDASetup.cmake

@@ -47,24 +47,22 @@ string(REPLACE ";" " " CUDA_ARCHITECTURES_STR "${CMAKE_CUDA_ARCHITECTURES}")
 cmessage(STATUS "Using following CUDA architectures: ${CUDA_ARCHITECTURES_STR}")
 
 if(NOT MaCh3_DEBUG_ENABLED)
-    target_compile_options(MaCh3CompileDefinitions INTERFACE
+    target_compile_options(MaCh3GPUCompilerOptions INTERFACE
         "$<$<COMPILE_LANGUAGE:CUDA>:-prec-sqrt=false;-use_fast_math;-O3;-Werror;cross-execution-space-call;-w>"
         "$<$<COMPILE_LANGUAGE:CUDA>:-Xptxas=-allow-expensive-optimizations=true;-Xptxas=-fmad=true;-Xptxas=-O3;>"
         "$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-fpic;-Xcompiler=-O3;-Xcompiler=-Wall;-Xcompiler=-Wextra;-Xcompiler=-Werror;-Xcompiler=-Wno-error=unused-parameter>"
     )
 else()
 #CW: -g and -G for debug flags to use cuda-gdb; slows stuff A LOT
 #-pxtas-options=-v, -maxregcount=N
-    target_compile_options(MaCh3CompileDefinitions INTERFACE
+    target_compile_options(MaCh3GPUCompilerOptions INTERFACE
         "$<$<COMPILE_LANGUAGE:CUDA>:-prec-sqrt=false;-use_fast_math;-Werror;cross-execution-space-call;-w>"
         "$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-g;>"
         "$<$<COMPILE_LANGUAGE:CUDA>:-Xptxas=-dlcm=ca;-Xptxas=-warn-lmem-usage;-Xptxas=-warn-spills;-Xptxas=-v;-Xcompiler=-Wall;-Xcompiler=-Wextra;-Xcompiler=-Werror;-Xcompiler=-Wno-error=unused-parameter>"
     )
-
-    target_compile_definitions(MaCh3CompileDefinitions INTERFACE "$<$<COMPILE_LANGUAGE:CUDA>:CUDA_ERROR_CHECK>")
+    target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE "$<$<COMPILE_LANGUAGE:CUDA>:CUDA_ERROR_CHECK>")
 endif()
-target_include_directories(MaCh3CompileDefinitions INTERFACE ${CUDAToolkit_INCLUDE_DIRS})
-
+target_include_directories(MaCh3GPUCompilerOptions INTERFACE ${CUDAToolkit_INCLUDE_DIRS})
 if(MaCh3_DEBUG_ENABLED)
   include(${CMAKE_CURRENT_LIST_DIR}/CUDASamples.cmake)
 endif()
@@ -76,8 +74,7 @@ if(NOT DEFINED NSplines_GPU)
 endif()
 
 # Pass NSplines_GPU as a preprocessor definition to the compiler
-target_compile_definitions(MaCh3CompileDefinitions INTERFACE NSplines_GPU=${NSplines_GPU})
-
+target_compile_definitions(MaCh3GPUCompilerOptions INTERFACE NSplines_GPU=${NSplines_GPU})
 cmessage(STATUS "Using \"${NSplines_GPU}\" for GPU EventByEvent Splines")
 
 

cmake/Modules/NuOscillatorSetup.cmake

@@ -53,15 +53,17 @@ IsTrue(MaCh3_MULTITHREAD_ENABLED DAN_USE_MULTITHREAD)
 IsTrue(MaCh3_LOW_MEMORY_STRUCTS_ENABLED DAN_DOUBLE)
 SwitchLogic(DAN_DOUBLE)
 
-# Get the compile options for MaCh3CompilerOptions
-get_target_property(compile_options MaCh3CompilerOptions INTERFACE_COMPILE_OPTIONS)
+# Get the CPU compile options for MaCh3CompilerOptions
+get_target_property(cpu_compile_options MaCh3CompilerOptions INTERFACE_COMPILE_OPTIONS)
 
 # Join the compile options list into a space-separated string
-string(REPLACE ";" " " compile_options_string "${compile_options}")
+string(REPLACE ";" " " compile_options_string "${cpu_compile_options}")
+
 
 #KS: This may seem hacky, but when CMAKE_CUDA_ARCHITECTURES is passed, it's treated as a string rather than a list. Since CMake uses semi-colon-delimited strings to represent lists, we convert it to a proper list to handle CUDA architectures correctly.
 set(CMAKE_CUDA_ARCHITECTURES_STRING ${CMAKE_CUDA_ARCHITECTURES})
-string(REPLACE " " ";" CMAKE_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
+#string(REPLACE " " ";" CMAKE_CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}")
+string(REPLACE " " ";" CMAKE_CUDA_ARCHITECTURES_STRING "${CMAKE_CUDA_ARCHITECTURES}")
 
 #Try adding Oscillator Class
 CPMAddPackage(
@@ -82,7 +84,7 @@ CPMAddPackage(
     "UseNuFASTLinear  ${USE_NuFastLiner}"
     "UseOscProb ${USE_OscProb}"
 
-    "NuOscillator_Compiler_Flags ${compile_options_string}"
+    "NuOscillator_Compiler_Flags ${cpu_compile_options}"
     "CMAKE_CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES_STRING}"
     "CMAKE_CXX_STANDARD ${CMAKE_CXX_STANDARD}"
 )

manager/CMakeLists.txt

@@ -22,10 +22,11 @@ set_target_properties(Manager PROPERTIES
 if(NOT CPU_ONLY)
     set_target_properties(Manager PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
     #KS: In cmake 3.18 this is no longer needed https://cmake.org/cmake/help/latest/variable/CMAKE_CUDA_ARCHITECTURES.html#cmake-cuda-architectures
-    set_property(TARGET Manager PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES})
+    set_property(TARGET Manager PROPERTY CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES}) 
 endif()
 
-target_link_libraries(Manager PUBLIC MaCh3CompilerOptions ROOT::Tree ROOT::Hist yaml-cpp spdlog)
+#If compiling with GPU is not enabled MaCh3GPUCompilerOptions will be empty
+target_link_libraries(Manager PUBLIC MaCh3CompilerOptions MaCh3GPUCompilerOptions yaml-cpp spdlog ROOT::Tree ROOT::Hist)
 target_link_libraries(Manager PRIVATE MaCh3Warnings)
 
 target_include_directories(Manager PUBLIC

splines/CMakeLists.txt

@@ -30,7 +30,7 @@ set_target_properties(Splines PROPERTIES
     PUBLIC_HEADER "${HEADERS}"
     EXPORT_NAME Splines)
 
-target_link_libraries(Splines PUBLIC Manager)
+target_link_libraries(Splines PUBLIC Manager MaCh3CompilerOptions MaCh3GPUCompilerOptions)
 target_link_libraries(Splines PRIVATE MaCh3Warnings)
 
 target_include_directories(Splines PUBLIC