diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b5eaa023d..6f454e7692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ Notable changes to QMCPACK are documented in this file. appears to work as expected. As a workaround, the CUDA toolkit 11.2 can be used. The actual NVIDIA drivers can be more recent. +* Enabled HIP as language in CMake (requires >= 3.21). When using HIP targeting AMD GPUs, + replace HIP_ARCH with CMAKE_HIP_ARCHITECTURES if HIP_ARCH was used to specify the GPU + architectures. + ## [3.15.0] - 2022-09-29 This is a recommended release for all users. There are many quality of life diff --git a/CMake/ClangCompilers.cmake b/CMake/ClangCompilers.cmake index ac439a4de8..b5c16c7f43 100644 --- a/CMake/ClangCompilers.cmake +++ b/CMake/ClangCompilers.cmake @@ -49,16 +49,16 @@ if(QMC_OMP) if(NOT DEFINED OFFLOAD_ARCH AND OFFLOAD_TARGET MATCHES "amdgcn") - if (DEFINED HIP_ARCH) - list(LENGTH HIP_ARCH NUMBER_HIP_ARCHITECTURES) - if(NUMBER_HIP_ARCHITECTURES EQUAL "1") - set(OFFLOAD_ARCH ${HIP_ARCH}) + if (DEFINED CMAKE_HIP_ARCHITECTURES) + list(LENGTH CMAKE_HIP_ARCHITECTURES NUMBER_CMAKE_HIP_ARCHITECTURESITECTURES) + if(NUMBER_CMAKE_HIP_ARCHITECTURESITECTURES EQUAL "1") + set(OFFLOAD_ARCH ${CMAKE_HIP_ARCHITECTURES}) else() message( FATAL_ERROR "LLVM does not yet support offload to multiple architectures! " - "Deriving OFFLOAD_ARCH from HIP_ARCH failed. " - "Please keep only one entry in HIP_ARCH or set OFFLOAD_ARCH.") + "Deriving OFFLOAD_ARCH from CMAKE_HIP_ARCHITECTURES failed. " + "Please keep only one entry in CMAKE_HIP_ARCHITECTURES or set OFFLOAD_ARCH.") endif() else() set(OFFLOAD_ARCH gfx906) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e16ba3e71..c49e519256 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -722,6 +722,9 @@ set(HAVE_CUDA 0) if(QMC_CUDA OR ENABLE_CUDA) if(QMC_CUDA2HIP) message(STATUS "CUDA2HIP enabled") # all the HIP and ROCm settings will be handled by ENABLE_ROCM + if(CMAKE_VERSION VERSION_LESS 3.21.0) + message(FATAL_ERROR "ENABLE_ROCM or QMC_CUDA2HIP require CMake 3.21.0 or later") + endif() else(QMC_CUDA2HIP) if(CMAKE_VERSION VERSION_LESS 3.18.0) message(FATAL_ERROR "QMC_CUDA or ENABLE_CUDA require CMake 3.18.0 or later") @@ -787,46 +790,44 @@ endif(USE_NVTX_API) # set up ROCM compiler options and libraries #------------------------------------------------------------------- if(ENABLE_ROCM) - if(ROCM_ROOT) - message(STATUS "ROCM_ROOT provided. Searching for FindHIP.cmake file") - find_path( - HIP_MODULE_FILE_DIR FindHIP.cmake - HINTS ${ROCM_ROOT} - PATH_SUFFIXES hip/cmake REQUIRED - NO_DEFAULT_PATH) - else() - message(STATUS "ROCM_ROOT not provided. Searching for FindHIP.cmake file.") - find_path( - HIP_MODULE_FILE_DIR FindHIP.cmake - HINTS $ENV{ROCM_PATH} /opt/rocm - PATH_SUFFIXES hip/cmake) - if(HIP_MODULE_FILE_DIR) - message(STATUS "Found FindHIP.cmake file. ROCM_ROOT will be derived.") - else() - message(FATAL_ERROR "Failed to find FindHIP.cmake file. Please set ROCM_ROOT to your ROCm installation.") - endif() - string(REGEX REPLACE "\/hip\/cmake" "" ROCM_ROOT "${HIP_MODULE_FILE_DIR}") + enable_language(HIP) + set(CMAKE_HIP_LINKER_PREFERENCE_PROPAGATES 0) + if(DEFINED ROCM_ROOT) + message(STATUS "Using ROCM_ROOT: ${ROCM_ROOT}") + # explicit ROCM_ROOT takes precedence + list(PREPEND CMAKE_PREFIX_PATH ${ROCM_ROOT}) + elseif(CMAKE_HIP_COMPILER_ROCM_ROOT) + list(PREPEND CMAKE_PREFIX_PATH ${CMAKE_HIP_COMPILER_ROCM_ROOT}) + endif() + + # use hip package to supply the hip runtime for CXX source code. + find_package(hip CONFIG) + if(NOT hip_FOUND) + message(FATAL_ERROR "The HIP runtime library cmake configure file hip-config.cmake cannot be found! " + "This happens when the ROCm installation cannot be found by CMake. " + "Rerun cmake with -DROCM_ROOT= added.") endif() - message(STATUS "Using ROCM_ROOT: ${ROCM_ROOT}") - - list(APPEND CMAKE_MODULE_PATH ${HIP_MODULE_FILE_DIR}) - list(APPEND CMAKE_PREFIX_PATH ${ROCM_ROOT}) - # need FindHIP for hip_add_library similar to deprecated FindCUDA style - find_package(HIP MODULE REQUIRED) - # need hip in config mode for hip::host target. - find_package(hip CONFIG REQUIRED) find_package(hipblas CONFIG REQUIRED) find_package(rocsolver CONFIG REQUIRED) find_package(rocthrust CONFIG REQUIRED) + # architecture flags - set(HIP_ARCH gfx906 CACHE STRING "HIP architecture gfxXXX") - list(APPEND HIP_HIPCC_FLAGS "-fPIC -ffast-math -O3 -std=c++17") - list(APPEND HIP_HIPCC_FLAGS "--amdgpu-target=${HIP_ARCH}") - list(APPEND HIP_HIPCC_FLAGS "--gpu-max-threads-per-block=256") + if(DEFINED HIP_ARCH) + unset(HIP_ARCH CACHE) + message( + FATAL_ERROR "HIP_ARCH option has been removed. Use -DCMAKE_HIP_ARCHITECTURES=gfx90a if -DHIP_ARCH=gfx90a was used.") + endif() + if(NOT DEFINED CMAKE_HIP_ARCHITECTURES) + set(CMAKE_HIP_ARCHITECTURES gfx906) + endif() + + set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -fPIC -ffast-math --gpu-max-threads-per-block=256") # warning suppression - list(APPEND HIP_HIPCC_FLAGS "-Wno-vla") - list(APPEND HIP_HIPCC_FLAGS "-Wno-deprecated-declarations") - list(APPEND HIP_HIPCC_FLAGS "-Wno-unused-command-line-argument") + set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -Wno-vla -Wno-deprecated-declarations -Wno-unused-command-line-argument") + + if(NOT DEFINED ROCM_ROOT) + list(PREPEND CMAKE_PREFIX_PATH ${CMAKE_HIP_COMPILER_ROCM_ROOT}) + endif() if(BUILD_AFQMC) find_package(rocrand CONFIG REQUIRED) diff --git a/config/build_olcf_crusher_afar.sh b/config/build_olcf_crusher_ROCm.sh similarity index 72% rename from config/build_olcf_crusher_afar.sh rename to config/build_olcf_crusher_ROCm.sh index 0b8b4d1014..2f659b3c46 100755 --- a/config/build_olcf_crusher_afar.sh +++ b/config/build_olcf_crusher_ROCm.sh @@ -4,21 +4,20 @@ # See https://github.com/QMCPACK/qmcpack/pull/4123 for more details on the module file if needed echo "Loading QMCPACK dependency modules for crusher" +module unload PrgEnv-gnu PrgEnv-cray PrgEnv-amd PrgEnv-gnu-amd PrgEnv-cray-amd +module unload amd amd-mixed gcc gcc-mixed cce cce-mixed +module load PrgEnv-amd amd/5.3.0 gcc-mixed/11.2.0 +module unload cray-libsci module load cmake/3.22.2 module load cray-fftw module load openblas/0.3.17-omp -module load boost/1.77.0-cxx17 -# private module until OLCF provides MPI compiler wrappers for afar compilers. -if [[ ! -d /ccs/proj/mat189/modules/crusher ]] ; then - echo "Required module folder /ccs/proj/mat189/modules/crusher not found!" - exit 1 -fi -module use /ccs/proj/mat189/modules/crusher -module load mpiwrappers/cray-mpich-afar module load cray-hdf5-parallel +module load boost/1.78.0 + +module list >& module_list.txt TYPE=Release -Compiler=afar +Compiler=rocm530 if [[ $# -eq 0 ]]; then source_folder=`pwd` @@ -52,7 +51,7 @@ if [[ $name == *"offload"* ]]; then fi if [[ $name == *"cuda2hip"* ]]; then - CMAKE_FLAGS="$CMAKE_FLAGS -DENABLE_CUDA=ON -DQMC_CUDA2HIP=ON -DHIP_ARCH=gfx90a" + CMAKE_FLAGS="$CMAKE_FLAGS -DENABLE_CUDA=ON -DQMC_CUDA2HIP=ON -DCMAKE_HIP_ARCHITECTURES=gfx90a" fi folder=build_crusher_${Compiler}_${name} @@ -63,7 +62,9 @@ echo "**********************************" mkdir $folder cd $folder if [ ! -f CMakeCache.txt ] ; then -cmake $CMAKE_FLAGS -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx $source_folder +cmake $CMAKE_FLAGS -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -DCMAKE_SYSTEM_NAME=CrayLinuxEnvironment \ + -DCMAKE_C_FLAGS=--gcc-toolchain=/opt/cray/pe/gcc/11.2.0/snos -DCMAKE_CXX_FLAGS=--gcc-toolchain=/opt/cray/pe/gcc/11.2.0/snos \ + $source_folder fi make -j16 cd .. diff --git a/config/build_olcf_summit_Clang.sh b/config/build_olcf_summit_Clang.sh index abcdf9232d..08a9970c3a 100755 --- a/config/build_olcf_summit_Clang.sh +++ b/config/build_olcf_summit_Clang.sh @@ -6,6 +6,7 @@ # How to invoke this script? # build_olcf_summit_Clang.sh # build all the variants assuming the current directory is the source directory. # build_olcf_summit_Clang.sh # build all the variants with a given source directory +# build_alcf_polaris_Clang.sh # build all the variants with a given source directory and install to echo "Purging current module set" module purge @@ -29,13 +30,19 @@ fi module use /gpfs/alpine/mat151/world-shared/opt/modules module load llvm/release-15.0.0-cuda11.0 +module list >& module_list.txt + TYPE=Release +Machine=summit Compiler=Clang if [[ $# -eq 0 ]]; then source_folder=`pwd` +elif [[ $# -eq 1 ]]; then + source_folder=$1 else source_folder=$1 + install_folder=$2 fi if [[ -f $source_folder/CMakeLists.txt ]]; then @@ -67,17 +74,30 @@ if [[ $name == *"cuda"* ]]; then CMAKE_FLAGS="$CMAKE_FLAGS -DENABLE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=70" fi -folder=build_summit_${Compiler}_${name} +folder=build_${Machine}_${Compiler}_${name} + +if [[ -v install_folder ]]; then + CMAKE_FLAGS="$CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX=$install_folder/$folder" +fi + echo "**********************************" echo "$folder" echo "$CMAKE_FLAGS" echo "**********************************" + mkdir $folder cd $folder + if [ ! -f CMakeCache.txt ] ; then -cmake $CMAKE_FLAGS -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx $source_folder + cmake $CMAKE_FLAGS -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx $source_folder fi -make -j16 + +if [[ -v install_folder ]]; then + make -j16 install && chmod -R -w $install_folder/$folder +else + make -j16 +fi + cd .. echo diff --git a/docs/installation.rst b/docs/installation.rst index 2fa0b0b8b0..b9c206b813 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -458,7 +458,7 @@ Similarly, HIP features can be enabled in conjunction with the offload code path :: - -D ENABLE_OFFLOAD=ON -D ENABLE_CUDA=ON -D QMC_CUDA2HIP=ON -DHIP_ARCH=gfx906 + -D ENABLE_OFFLOAD=ON -D ENABLE_CUDA=ON -D QMC_CUDA2HIP=ON -DCMAKE_HIP_ARCHITECTURES=gfx906 Similarly, SYCL features can be enabled in conjunction with the offload code path to improve performance on Intel GPUs. diff --git a/src/AFQMC/CMakeLists.txt b/src/AFQMC/CMakeLists.txt index 79fe25a79e..c203bb8f37 100644 --- a/src/AFQMC/CMakeLists.txt +++ b/src/AFQMC/CMakeLists.txt @@ -103,8 +103,8 @@ if(ENABLE_CUDA) add_library(afqmc ${AFQMC_SRCS}) target_link_libraries(afqmc PRIVATE CUDA::curand CUDA::cusparse CUDA::cusolver CUDA::cublas) elseif(ENABLE_HIP) - set_source_files_properties(${AFQMC_HIP_SRCS} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1) - hip_add_library(afqmc_hip_lib ${AFQMC_HIP_SRCS}) + set_source_files_properties(${AFQMC_HIP_SRCS} PROPERTIES LANGUAGE HIP) + add_library(afqmc_hip_lib ${AFQMC_HIP_SRCS}) target_link_libraries(afqmc_hip_lib PUBLIC HIP::HIP ROCM::libraries Boost::boost qmc_external_thrust) add_library(afqmc ${AFQMC_SRCS}) target_link_libraries(afqmc PUBLIC afqmc_hip_lib) diff --git a/src/Particle/CMakeLists.txt b/src/Particle/CMakeLists.txt index 0aab15e218..6f6741bedc 100644 --- a/src/Particle/CMakeLists.txt +++ b/src/Particle/CMakeLists.txt @@ -68,11 +68,10 @@ target_compile_options(qmcparticle_omptarget PRIVATE "$<$> $GITHUB_ENV - + cmake -GNinja \ -DCMAKE_C_COMPILER=/opt/rocm/llvm/bin/clang \ -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ diff --git a/tests/test_automation/nightly_test_scripts/nightly_anl_bora.sh b/tests/test_automation/nightly_test_scripts/nightly_anl_bora.sh index e4e0c65f0b..94663b22b0 100755 --- a/tests/test_automation/nightly_test_scripts/nightly_anl_bora.sh +++ b/tests/test_automation/nightly_test_scripts/nightly_anl_bora.sh @@ -107,10 +107,10 @@ elif [[ $sys == *"ROCm"* ]]; then module load openmpi/4.0.2-llvm if [[ $sys == *"Offload-CUDA2HIP"* ]]; then - CTEST_FLAGS="$CTEST_FLAGS -DQMC_OPTIONS='-DHIP_ARCH=gfx1030;-DENABLE_OFFLOAD=ON;-DQMC_PERFORMANCE_NIO_MAX_ATOMS=32'" + CTEST_FLAGS="$CTEST_FLAGS -DQMC_OPTIONS='-DCMAKE_HIP_ARCHITECTURES=gfx1030;-DENABLE_OFFLOAD=ON;-DQMC_PERFORMANCE_NIO_MAX_ATOMS=32'" CTEST_FLAGS="$CTEST_FLAGS -DENABLE_CUDA=ON -DQMC_CUDA2HIP=ON" elif [[ $sys == *"CUDA2HIP"* ]]; then - CTEST_FLAGS="$CTEST_FLAGS -DQMC_OPTIONS='-DHIP_ARCH=gfx1030;-DQMC_PERFORMANCE_NIO_MAX_ATOMS=32'" + CTEST_FLAGS="$CTEST_FLAGS -DQMC_OPTIONS='-DCMAKE_HIP_ARCHITECTURES=gfx1030;-DQMC_PERFORMANCE_NIO_MAX_ATOMS=32'" CTEST_FLAGS="$CTEST_FLAGS -DENABLE_CUDA=ON -DQMC_CUDA2HIP=ON" elif [[ $sys == *"Offload"* ]]; then CTEST_FLAGS="$CTEST_FLAGS -DQMC_OPTIONS='-DENABLE_OFFLOAD=ON;-DOFFLOAD_TARGET=amdgcn-amd-amdhsa;-DOFFLOAD_ARCH=gfx1030;-DQMC_PERFORMANCE_NIO_MAX_ATOMS=32'" diff --git a/tests/test_automation/nightly_test_scripts/nightly_ornl.sh b/tests/test_automation/nightly_test_scripts/nightly_ornl.sh index 07a195da8e..f030ad5dcc 100755 --- a/tests/test_automation/nightly_test_scripts/nightly_ornl.sh +++ b/tests/test_automation/nightly_test_scripts/nightly_ornl.sh @@ -393,7 +393,7 @@ if [[ $sys == *"legacycu2hip"* ]]; then export ROCM_PATH=/opt/rocm export PATH=${PATH}:${ROCM_PATH}/bin:${ROCM_PATH}/opencl/bin QMCPACK_TEST_SUBMIT_NAME=${QMCPACK_TEST_SUBMIT_NAME}-Legacy-CUDA2HIP - QMC_OPTIONS="${QMC_OPTIONS};-DENABLE_CUDA=ON;-DQMC_CUDA2HIP=ON;-DHIP_ARCH=${amdgpuarch}" + QMC_OPTIONS="${QMC_OPTIONS};-DENABLE_CUDA=ON;-DQMC_CUDA2HIP=ON;-DCMAKE_HIP_ARCHITECTURES=${amdgpuarch}" fi # MKL