Cleaning up config for CUDA and GCC and openmp offload

UO-OACISS · Aug 12, 2021 · 4a204e8 · 4a204e8
1 parent 686b402
commit 4a204e8
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 11 deletions.
diff --git a/src/openmp/CMakeLists.txt b/src/openmp/CMakeLists.txt
@@ -10,18 +10,36 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 
 if (OpenMP_FOUND) # don't think we'd be in here otherwise...
     if (CUPTI_FOUND)
-        #set (CLANG_OPENMP_NVPTX_DEFAULT_ARCH "sm_60")
-        #set (LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES "35,60,70")
-        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS} -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvidia-cuda")
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvidia-cuda")
+        if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND
+            CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.0)
+            set(OMPT_TARGET_TEST ompt_target ompt_target_matmult ompt_target_daxpy ompt_target_vector_add)
+        elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+            set(omp_target_flags "${OpenMP_C_FLAGS} -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvidia-cuda")
+            check_cxx_compiler_flag("${omp_target_flags}" flag_supported)
+            if(flag_supported)
+                message(INFO " ${omp_target_flags} supported by compiler, enabling it")
+                set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${omp_target_flags}")
+                set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${omp_target_flags}")
+                set(OMPT_TARGET_TEST ompt_target ompt_target_matmult ompt_target_daxpy ompt_target_vector_add)
+            else()
+                set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+                set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+            endif()
+            unset(flag_supported CACHE)
+        endif()
     endif (CUPTI_FOUND)
     if (ROCTRACER_FOUND)
-        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS} -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx906")
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS} -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx906")
+        set (omp_target_flags "${OpenMP_C_FLAGS} -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx906")
+        check_cxx_compiler_flag("${omp_target_flags}" flag_supported)
+        if(flag_supported)
+            set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${omp_target_flags")
+            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${omp_target_flags")
+            set(OMPT_TARGET_TEST ompt_target ompt_target_matmult ompt_target_daxpy ompt_target_vector_add)
+        else()
+            set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+        endif()
     endif (ROCTRACER_FOUND)
-    if (ROCTRACER_FOUND OR CUPTI_FOUND)
-        set(OMPT_TARGET_TEST ompt_target ompt_target_matmult ompt_target_daxpy ompt_target_vector_add)
-    endif (ROCTRACER_FOUND OR CUPTI_FOUND)
 endif (OpenMP_FOUND)
 
 set(example_programs

diff --git a/src/scripts/apex_exec b/src/scripts/apex_exec
@@ -128,6 +128,7 @@ while (( "$#" )); do
       ;;
     --apex:pthread)
       pthread=yes
+      export APEX_TIME_TOP_LEVEL_OS_THREADS=1
       shift
       ;;
     --apex:memory)
@@ -344,8 +345,8 @@ else
     APEX_LD_PRELOAD=${PTHREAD_LIB}${MEMORY_LIB}${BASEDIR}/lib/libapex${SHLIBX}:${OMPT_LIB}:${LD_PRELOAD}
 fi
 
-# remove double colons
-APEX_LD_PRELOAD=`echo ${APEX_LD_PRELOAD} | sed -e "s/::/:/g" -e "s/:$//"`
+# remove spaces and double colons
+APEX_LD_PRELOAD=`echo ${APEX_LD_PRELOAD} | sed -e "s/ /:/g" -e "s/::/:/g" -e "s/:$//"`
 export CPUPROFILE=two_vars.prof
 
 if [ ${apple} = 1 ]; then

diff --git a/src/unit_tests/CUDA/CMakeLists.txt b/src/unit_tests/CUDA/CMakeLists.txt
@@ -47,6 +47,8 @@ foreach(example_program ${example_programs})
   add_dependencies (tests "${example_program}_cu")
   add_test ("test_${example_program}_cu" "${example_program}_cu")
   # install(TARGETS "${example_program}_cu" RUNTIME DESTINATION "bin/apex_unit_tests" OPTIONAL)
+  set_tests_properties("test_${example_program}_cu" PROPERTIES ENVIRONMENT "APEX_ENABLE_CUDA=1;APEX_CUDA_KERNEL_DETAILS=1;APEX_CUDA_COUNTERS=1;APEX_CUDA_DRIVER_API=1")
+
 endforeach()
 
 # Make sure the linker can find the Apex library once it is built.