Start migrating to Intel oneMKL

CMakeLists.txt

@@ -34,7 +34,8 @@ endif()
 
 option(BUILD_SHARED_LIBS "Build shared libraries." ON)
 option(DLAF_WITH_OPENMP "${DLAF_WITH_OPENMP_DESCRIPTION}" ${DLAF_WITH_OPENMP_DEFAULT})
-option(DLAF_WITH_MKL "Enable MKL as provider for LAPACK" OFF)
+option(DLAF_WITH_MKL "Enable oneMKL as provider for BLAS and LAPACK" OFF)
+option(DLAF_WITH_MKL_LEGACY "Enable MKL as provider for BLAS and  LAPACK" OFF)
 option(DLAF_WITH_CUDA "Enable CUDA support" OFF)
 option(DLAF_WITH_HIP "Enable HIP support" OFF)
 cmake_dependent_option(DLAF_WITH_CUDA_MPI_RDMA "Enable MPI CUDA RDMA" OFF "DLAF_WITH_CUDA" OFF)
@@ -48,7 +49,11 @@ option(DLAF_BUILD_DOC "Build documentation" OFF)
 option(DLAF_WITH_PRECOMPILED_HEADERS "Use precompiled headers." OFF)
 option(DLAF_WITH_SCALAPACK "Build ScaLAPACK-like C API (requires ScaLAPACK)" OFF)
 
-if(DLAF_WITH_MKL)
+if(DLAF_WITH_MKL AND DLAF_WITH_MKL_LEGACY)
+  message(SEND_ERROR "DLAF_WITH_MKL and DLAF_WITH_MKL_LEGACY are mutually exclusive.")
+endif()
+
+if(DLAF_WITH_MKL OR DLAF_WITH_MKL_LEGACY)
   # When using MKL there is no need to set the number of threads with
   # omp_set_num_threads; it's sufficient to use MKL's own mechanisms.
   set(DLAF_WITH_OPENMP OFF CACHE BOOL "${DLAF_WITH_OPENMP_DESCRIPTION}" FORCE)
@@ -130,8 +135,23 @@ if(DLAF_WITH_OPENMP)
 endif()
 
 # ----- LAPACK/SCALAPACK
-if(DLAF_WITH_MKL)
-  find_package(MKL REQUIRED)
+if(DLAF_WITH_MKL) # oneMKL
+  set(MKL_INTERFACE "lp64" CACHE STRING "")
+  set(MKL_THREADING "sequential" CACHE STRING "")
+  set(MKL_MPI "mpich" CACHE STRING "")
+
+  find_package(MKL CONFIG REQUIRED)
+
+  set(LAPACK_FOUND TRUE)
+  add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL)
+  target_link_libraries(DLAF::LAPACK INTERFACE MKL::MKL)
+
+  if(DLAF_WITH_SCALAPACK)
+    set(SCALAPACK_FOUND TRUE)
+    add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL)
+  endif()
+elseif(DLAF_WITH_MKL_LEGACY) # MKL (deprecated)
+  find_package(MKL MODULE REQUIRED)
 
   set(MKL_LAPACK_TARGET "mkl::mkl_intel_32bit_seq_dyn"
       CACHE STRING "MKL LAPACK target (see FindMKL for details)"

cmake/template/DLAFConfig.cmake.in

@@ -20,6 +20,7 @@ list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR})
 # ===== VARIABLES
 set(DLAF_WITH_OPENMP @DLAF_WITH_OPENMP@)
 set(DLAF_WITH_MKL @DLAF_WITH_MKL@)
+set(DLAF_WITH_MKL_LEGACY @DLAF_WITH_MKL_LEGACY@)
 set(DLAF_WITH_CUDA @DLAF_WITH_CUDA@)
 set(DLAF_WITH_HIP @DLAF_WITH_HIP@)
 set(DLAF_WITH_GPU @DLAF_WITH_GPU@)
@@ -53,6 +54,19 @@ endif()
 
 # ----- LAPACK
 if(DLAF_WITH_MKL)
+  set(MKL_INTERFACE "@MKL_INTERFACE@")
+  set(MKL_THREADING "@MKL_THREADING@")
+  set(MKL_MPI "@MKL_MPI@")
+
+  find_dependency(MKL)
+
+  add_library(DLAF::LAPACK INTERFACE IMPORTED GLOBAL)
+  target_link_libraries(DLAF::LAPACK INTERFACE MKL::MKL)
+
+  if(DLAF_WITH_SCALAPACK)
+    add_library(DLAF::SCALAPACK INTERFACE IMPORTED GLOBAL)
+  endif()
+elseif(DLAF_WITH_MKL_LEGACY)
   set(MKL_ROOT "@MKL_ROOT@")
   set(MKL_CUSTOM_THREADING "@MKL_THREADING@")
 

spack/packages/dla-future/package.py

@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 # dlaf-no-license-check
-
 from spack.package import *
 
 
@@ -136,18 +135,43 @@ def cmake_args(self):
         args.append(self.define_from_variant("BUILD_SHARED_LIBS", "shared"))
 
         # BLAS/LAPACK
-        if self.spec["lapack"].name in INTEL_MATH_LIBRARIES:
+        if spec["lapack"].name in INTEL_MATH_LIBRARIES:
+            mkl_provider = spec["lapack"].name
+
             vmap = {
-                "none": "seq",
-                "openmp": "omp",
-                "tbb": "tbb",
-            }  # Map MKL variants to LAPACK target name
-            mkl_threads = vmap[spec["intel-mkl"].variants["threads"].value]
-            # TODO: Generalise for intel-oneapi-mkl
-            args += [
-                self.define("DLAF_WITH_MKL", True),
-                self.define("MKL_LAPACK_TARGET", f"mkl::mkl_intel_32bit_{mkl_threads}_dyn"),
-            ]
+                "intel-oneapi-mkl": {
+                    "threading": {
+                        "none": "sequential",
+                        "openmp": "gnu_thread",
+                        "tbb": "tbb_thread",
+                    },
+                    "mpi": {"mpich": "intelmpi", "openmpi": "openmpi"},
+                },
+                "intel-mkl": {
+                    "threading": {"none": "seq", "openmp": "omp", "tbb": "tbb"},
+                    "mpi": {"mpich": "mpich", "openmpi": "ompi"},
+                },
+            }
+
+            if mkl_provider not in vmap.keys():
+                raise RuntimeError(
+                    f"dla-future does not support {mkl_provider} as lapack provider"
+                )
+            mkl_mapper = vmap[mkl_provider]
+
+            mkl_threads = mkl_mapper["threading"][spec[mkl_provider].variants["threads"].value]
+            if mkl_provider == "intel-oneapi-mkl":
+                args += [
+                    self.define("DLAF_WITH_MKL", True),
+                    self.define("MKL_INTERFACE", "lp64"),
+                    self.define("MKL_THREADING", mkl_threads),
+                ]
+            elif mkl_provider == "intel-mkl":
+                args += [
+                    self.define("DLAF_WITH_MKL_LEGACY", True),
+                    self.define("MKL_LAPACK_TARGET", f"mkl::mkl_intel_32bit_{mkl_threads}_dyn"),
+                ]
+
             if "+scalapack" in spec:
                 if (
                     "^mpich" in spec
@@ -156,15 +180,24 @@ def cmake_args(self):
                     or "^mvapich" in spec
                     or "^mvapich2" in spec
                 ):
-                    mkl_mpi = "mpich"
+                    mkl_mpi = mkl_mapper["mpi"]["mpich"]
                 elif "^openmpi" in spec:
-                    mkl_mpi = "ompi"
-                args.append(
-                    self.define(
-                        "MKL_SCALAPACK_TARGET",
-                        f"mkl::scalapack_{mkl_mpi}_intel_32bit_{mkl_threads}_dyn",
+                    mkl_mpi = mkl_mapper["mpi"]["openmpi"]
+                else:
+                    raise RuntimeError(
+                        f"dla-future does not support {spec['mpi'].name} as mpi provider with "
+                        f"the selected scalapack provider {mkl_provider}"
+                    )
+
+                if mkl_provider == "intel-oneapi-mkl":
+                    args.append(self.define("MKL_MPI", mkl_mpi))
+                elif mkl_provider == "intel-mkl":
+                    args.append(
+                        self.define(
+                            "MKL_SCALAPACK_TARGET",
+                            f"mkl::scalapack_{mkl_mpi}_intel_32bit_{mkl_threads}_dyn",
+                        )
                     )
-                )
         else:
             args.append(self.define("DLAF_WITH_MKL", False))
             args.append(
@@ -182,12 +215,12 @@ def cmake_args(self):
         args.append(self.define_from_variant("DLAF_WITH_CUDA", "cuda"))
         args.append(self.define_from_variant("DLAF_WITH_HIP", "rocm"))
         if "+rocm" in spec:
-            archs = self.spec.variants["amdgpu_target"].value
+            archs = spec.variants["amdgpu_target"].value
             if "none" not in archs:
                 arch_str = ";".join(archs)
                 args.append(self.define("CMAKE_HIP_ARCHITECTURES", arch_str))
         if "+cuda" in spec:
-            archs = self.spec.variants["cuda_arch"].value
+            archs = spec.variants["cuda_arch"].value
             if "none" not in archs:
                 arch_str = ";".join(archs)
                 args.append(self.define("CMAKE_CUDA_ARCHITECTURES", arch_str))
@@ -199,12 +232,12 @@ def cmake_args(self):
         args.append(self.define_from_variant("DLAF_BUILD_DOC", "doc"))
 
         ### For the spack repo only the else branch should remain.
-        if "+ci-test" in self.spec:
+        if "+ci-test" in spec:
             # Enable TESTS and setup CI specific parameters
             args.append(self.define("CMAKE_CXX_FLAGS", "-Werror"))
-            if "+cuda" in self.spec:
+            if "+cuda" in spec:
                 args.append(self.define("CMAKE_CUDA_FLAGS", "-Werror=all-warnings"))
-            if "+rocm" in self.spec:
+            if "+rocm" in spec:
                 args.append(self.define("CMAKE_HIP_FLAGS", "-Werror"))
             args.append(self.define("BUILD_TESTING", True))
             args.append(self.define("DLAF_BUILD_TESTING", True))
@@ -215,7 +248,7 @@ def cmake_args(self):
             args.append(self.define("DLAF_BUILD_TESTING", self.run_tests))
 
         ### Variants available only in the DLAF repo spack package
-        if "+ci-check-threads" in self.spec:
+        if "+ci-check-threads" in spec:
             args.append(self.define("DLAF_TEST_PREFLAGS", "check-threads"))
         ###
 

src/CMakeLists.txt

@@ -94,6 +94,7 @@ target_compile_definitions(
             $<$<BOOL:${DLAF_ASSERT_HEAVY_ENABLE}>:DLAF_ASSERT_HEAVY_ENABLE>
             DLAF_FUNCTION_NAME=$<IF:$<BOOL:is_pretty_function_available>,__PRETTY_FUNCTION__,__func__>
             $<$<BOOL:${DLAF_WITH_MKL}>:DLAF_WITH_MKL>
+            $<$<BOOL:${DLAF_WITH_MKL_LEGACY}>:DLAF_WITH_MKL>
             $<$<BOOL:${DLAF_WITH_OPENMP}>:DLAF_WITH_OPENMP>
             $<$<BOOL:${DLAF_WITH_GPU}>:DLAF_WITH_GPU>
             $<$<BOOL:${DLAF_WITH_CUDA}>:DLAF_WITH_CUDA>