Bug fixes for GPU builds

CMakeLists.txt

@@ -50,6 +50,8 @@ if(PALACE_WITH_CUDA AND PALACE_WITH_HIP)
   message(FATAL_ERROR "PALACE_WITH_CUDA is not compatible with PALACE_WITH_HIP")
 endif()
 if(PALACE_WITH_CUDA)
+  # Note: The new behavior of CMake policy CMP0104 will initialize CMAKE_CUDA_ARCHITECTURES
+  # to an (old) compatible value even when not set by the user.
   enable_language(CUDA)
   get_filename_component(NVCC_DIR ${CMAKE_CUDA_COMPILER} DIRECTORY)
   get_filename_component(BIN_NVCC_DIR ${NVCC_DIR} DIRECTORY)

cmake/ExternalGitTags.cmake

@@ -132,7 +132,7 @@ set(EXTERN_MFEM_GIT_BRANCH
   "Git branch for external MFEM build"
 )
 set(EXTERN_MFEM_GIT_TAG
-  "c444b17c973cc301590a6ac186fb33587b5881e6" CACHE STRING  # master @ 05/18/2024
+  "9a327eeca6472254e8bdefc1527aa130250e528e" CACHE STRING  # master @ 05/18/2024
   "Git tag for external MFEM build"
 )
 

cmake/ExternalMAGMA.cmake

@@ -71,10 +71,14 @@ if(PALACE_WITH_HIP)
   endif()
 endif()
 set(MAKE_GENERATE_INC "${MAKE_GENERATE_INC}FORT = true\\n")
+file(WRITE ${CMAKE_BINARY_DIR}/extern/magma-cmake/make.inc.cmake
+  "file(WRITE make.inc \"${MAKE_GENERATE_INC}\")\n"
+)
 
 string(REPLACE ";" "; " MAGMA_OPTIONS_PRINT "${MAGMA_OPTIONS}")
 message(STATUS "MAGMA_OPTIONS: ${MAGMA_OPTIONS_PRINT}")
 
+# Note: MAGMA requires Python < 3.12 for patch step as of June 2024
 include(ExternalProject)
 ExternalProject_Add(magma
   DEPENDS           ${MAGMA_DEPENDENCIES}
@@ -86,7 +90,7 @@ ExternalProject_Add(magma
   PREFIX            ${CMAKE_BINARY_DIR}/extern/magma-cmake
   UPDATE_COMMAND    ""
   PATCH_COMMAND
-    echo -e "${MAKE_GENERATE_INC}" > make.inc &&
+    ${CMAKE_COMMAND} -P ../magma-cmake/make.inc.cmake &&
     ${CMAKE_MAKE_PROGRAM} generate
   CONFIGURE_COMMAND ${CMAKE_COMMAND} <SOURCE_DIR> "${MAGMA_OPTIONS}"
   TEST_COMMAND      ""

cmake/ExternalMETIS.cmake

@@ -99,9 +99,11 @@ if(PALACE_WITH_SUPERLU OR PALACE_WITH_STRUMPACK)
     message(FATAL_ERROR "MPI is not found when trying to build ParMETIS")
   endif()
   if(NOT CMAKE_C_COMPILER STREQUAL MPI_C_COMPILER)
+    string(REPLACE ";" "$<SEMICOLON>" PARMETIS_MPI_LIBRARIES "${MPI_C_LIBRARIES}")
+    string(REPLACE ";" "$<SEMICOLON>" PARMETIS_MPI_INCLUDE_DIRS "${MPI_C_INCLUDE_DIRS}")
     list(APPEND PARMETIS_OPTIONS
-      "-DMPI_LIBRARIES=${MPI_C_LIBRARIES}"
-      "-DMPI_INCLUDE_PATH=${MPI_C_INCLUDE_DIRS}"
+      "-DMPI_LIBRARIES=${PARMETIS_MPI_LIBRARIES}"
+      "-DMPI_INCLUDE_PATH=${PARMETIS_MPI_INCLUDE_DIRS}"
     )
   endif()
 

cmake/ExternalMFEM.cmake

@@ -164,22 +164,18 @@ if(PALACE_BUILD_EXTERNAL_DEPS)
     )
   endif()
 
-  # HYPRE is built with cusparse, curand (or HIP counterparts).
+  # HYPRE is built with cusparse, curand (or HIP counterparts), and these are added to
+  # HYPRE_LIBRARIES by the MFEM CMake build. However, this ignores the include directories
+  # (for #include <cusparse.h>, for example), which we can add this way via CMake defining
+  # CUDAToolkit_INCLUDE_DIRS (and the HIP counterpart).
   if(PALACE_WITH_CUDA)
-    find_package(CUDAToolkit REQUIRED)
-    get_target_property(HYPRE_CURAND_LIBRARY CUDA::curand LOCATION)
-    get_target_property(HYPRE_CUSPARSE_LIBRARY CUDA::cusparse LOCATION)
     list(APPEND MFEM_OPTIONS
-      "-DHYPRE_REQUIRED_LIBRARIES=${HYPRE_CURAND_LIBRARY}$<SEMICOLON>${HYPRE_CUSPARSE_LIBRARY}"
+      "-DHYPRE_REQUIRED_PACKAGES=CUDAToolkit"
     )
   endif()
   if(PALACE_WITH_HIP)
-    find_package(rocrand REQUIRED)
-    find_package(rocsparse REQUIRED)
-    get_target_property(HYPRE_ROCRAND_LIBRARY roc::rocrand LOCATION)
-    get_target_property(HYPRE_ROCSPARSE_LIBRARY roc::rocsparse LOCATION)
     list(APPEND MFEM_OPTIONS
-      "-DHYPRE_REQUIRED_LIBRARIES=${HYPRE_ROCRAND_LIBRARY}$<SEMICOLON>${HYPRE_ROCSPARSE_LIBRARY}"
+      "-DHYPRE_REQUIRED_PACKAGES=rocsparse"
     )
   endif()
 
@@ -241,11 +237,11 @@ Intel C++ compiler for MUMPS and STRUMPACK dependencies")
       list(APPEND SUPERLU_REQUIRED_PACKAGES "OpenMP")
     endif()
     if(PALACE_WITH_CUDA)
-      list(APPEND SUPERLU_REQUIRED_PACKAGES "CUDA")
+      list(APPEND SUPERLU_REQUIRED_PACKAGES "CUDAToolkit")
       list(APPEND SUPERLU_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_CUDA_LIBRARIES})
     endif()
     if(PALACE_WITH_HIP)
-      list(APPEND SUPERLU_REQUIRED_PACKAGES "HIP")
+      list(APPEND SUPERLU_REQUIRED_PACKAGES "hipblas$<SEMICOLON>rocblas")
       list(APPEND SUPERLU_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_ROCM_LIBRARIES})
     endif()
     string(REPLACE ";" "$<SEMICOLON>" SUPERLU_REQUIRED_PACKAGES "${SUPERLU_REQUIRED_PACKAGES}")
@@ -272,11 +268,11 @@ Intel C++ compiler for MUMPS and STRUMPACK dependencies")
       list(APPEND STRUMPACK_REQUIRED_PACKAGES "OpenMP")
     endif()
     if(PALACE_WITH_CUDA)
-      list(APPEND STRUMPACK_REQUIRED_PACKAGES "CUDA")
+      list(APPEND STRUMPACK_REQUIRED_PACKAGES "CUDAToolkit")
       list(APPEND STRUMPACK_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_CUDA_LIBRARIES})
     endif()
     if(PALACE_WITH_HIP)
-      list(APPEND STRUMPACK_REQUIRED_PACKAGES "HIP")
+      list(APPEND STRUMPACK_REQUIRED_PACKAGES "hipblas$<SEMICOLON>rocblas")
       list(APPEND STRUMPACK_REQUIRED_LIBRARIES ${SUPERLU_STRUMPACK_ROCM_LIBRARIES})
     endif()
     string(REPLACE ";" "$<SEMICOLON>" STRUMPACK_REQUIRED_PACKAGES "${STRUMPACK_REQUIRED_PACKAGES}")
@@ -364,7 +360,6 @@ set(MFEM_PATCH_FILES
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_mesh_part_const.diff"
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_par_tet_mesh_fix_dev.diff"
   "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_gmsh_parser_performance.diff"
-  "${CMAKE_SOURCE_DIR}/extern/patch/mfem/patch_nc_internal_bdr_project_fix.diff"
 )
 
 include(ExternalProject)

cmake/ExternalPalace.cmake

@@ -80,9 +80,6 @@ string(REPLACE ";" "; " PALACE_OPTIONS_PRINT "${PALACE_OPTIONS}")
 message(STATUS "PALACE_OPTIONS: ${PALACE_OPTIONS_PRINT}")
 
 include(ExternalProject)
-if(POLICY CMP0114)
-  cmake_policy(SET CMP0114 NEW)
-endif()
 ExternalProject_Add(palace
   DEPENDS           ${PALACE_DEPENDENCIES}
   SOURCE_DIR        ${CMAKE_SOURCE_DIR}/palace