Merge pull request #87 from rebecca-fosdick/release/ccl_2021.8

Intel(R) oneAPI Collective Communications Library (oneCCL) 2021.8

CMakeLists.txt

@@ -44,16 +44,35 @@ if (${CMAKE_BUILD_TYPE_CASE_INSENSITIVE} STREQUAL "debug")
     set(USE_SECURITY_FLAGS FALSE)
 endif()
 
+if ("${DRM_INCLUDE_DIR}" STREQUAL "")
+    find_path(DRM_INCLUDE_DIR "i915_drm.h" PATH_SUFFIXES "drm" "libdrm")
+else()
+    message(STATUS "DRM_INCLUDE_DIR is set by user: ${DRM_INCLUDE_DIR}")
+endif()
+
+if (DRM_INCLUDE_DIR)
+    if (EXISTS ${DRM_INCLUDE_DIR}/i915_drm.h)
+        set(ENABLE_DRM TRUE)
+    else()
+        set(ENABLE_DRM FALSE)
+    endif()
+else()
+    set(ENABLE_DRM FALSE)
+endif()
+
 option(BUILD_EXAMPLES "Build examples" TRUE)
 option(BUILD_FT "Build functional tests" TRUE)
+option(BUILD_REG_TESTS "Build regression tests" TRUE)
 option(BUILD_CONFIG "Build cmake configs" TRUE)
 option(ENABLE_MPI "Enable MPI support" TRUE)
 option(ENABLE_MPI_TESTS "Enable MPI tests support" TRUE)
 option(ENABLE_SYCL_INTEROP_EVENT "Enable SYCL interop event support" TRUE)
-option(ENABLE_OFI_HMEM "Enable support OFI HMEM support" FALSE)
+option(ENABLE_OFI_HMEM "Enable OFI HMEM support" TRUE)
 option(ENABLE_OFI_OOT_PROV "Enable OFI out-of-tree providers support" FALSE)
 option(ENABLE_ITT "Enable ITT profiling support" TRUE)
+option(ENABLE_PMIX "Enable PMIX support" TRUE)
 option(ENABLE_STUB_BACKEND "Enable stub backend" TRUE)
+option(ENABLE_LINKER_RUNPATH "Enable linker runpath flags" FALSE)
 
 option(USE_CODECOV_FLAGS "Calculate code coverage" FALSE)
 option(WITH_ASAN "Use address sanitizer, can only be used in Debug build" FALSE)
@@ -79,7 +98,10 @@ message(STATUS "Enable SYCL interop event support: ${ENABLE_SYCL_INTEROP_EVENT}"
 message(STATUS "Enable OFI HMEM support: ${ENABLE_OFI_HMEM}")
 message(STATUS "Enable OFI out-of-tree providers support: ${ENABLE_OFI_OOT_PROV}")
 message(STATUS "Enable ITT profiling support: ${ENABLE_ITT}")
-message(STATUS "Enable stub backend" ${ENABLE_STUB_BACKEND})
+message(STATUS "Enable PMIX support: ${ENABLE_PMIX}")
+message(STATUS "Enable DRM support: ${ENABLE_DRM}")
+message(STATUS "Enable stub backend: ${ENABLE_STUB_BACKEND}")
+message(STATUS "Enable linker rpath flags: ${ENABLE_LINKER_RUNPATH}")
 
 add_definitions(-DCCL_C_COMPILER="${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
 add_definitions(-DCCL_CXX_COMPILER="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
@@ -131,6 +153,11 @@ message(STATUS "ITT_LIB_DIR: ${ITT_LIB_DIR}")
 set(LEVEL_ZERO_INCLUDE_DIR "${DEPS_DIR}/level_zero/include/")
 message(STATUS "LEVEL_ZERO_INCLUDE_DIR: ${LEVEL_ZERO_INCLUDE_DIR}")
 
+message(STATUS "DRM_INCLUDE_DIR: ${DRM_INCLUDE_DIR}")
+
+set(PMIX_INCLUDE_DIR "${DEPS_DIR}/pmix/include/")
+message(STATUS "PMIX_INCLUDE_DIR: ${PMIX_INCLUDE_DIR}")
+
 set(CMAKE_SKIP_INSTALL_RPATH TRUE)
 set(CMAKE_SKIP_RPATH TRUE)
 
@@ -141,15 +168,21 @@ if (${CMAKE_VERSION} VERSION_LESS 3.1)
     set(C_COMPILER_FLAGS "-std=gnu99")
 endif()
 
+if (${CMAKE_C_COMPILER_ID} STREQUAL "Clang" AND ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
+    set(EXTRA_WARN_FLAGS "-Wshadow")
+endif()
+
+set(COMPILER_WARN_FLAGS "-Wall -Wextra -Wno-unused-parameter -Werror ${EXTRA_WARN_FLAGS}")
+
 # common release/debug compilation settings
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_COMPILER_FLAGS} -Wall -Wextra -Wno-unused-parameter -Werror -D_GNU_SOURCE -fvisibility=internal")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${C_COMPILER_FLAGS} ${COMPILER_WARN_FLAGS} -D_GNU_SOURCE -fvisibility=internal")
 set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${C_COMPILER_FLAGS} -O0 -g -DENABLE_DEBUG")
 set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${C_COMPILER_FLAGS} -O3")
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${C_COMPILER_FLAGS} -O2 -g")
 set(CMAKE_C_STANDARD 99)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMPILER_FLAGS} -Wall -Wextra -Wno-unused-parameter -Werror -D_GNU_SOURCE -fvisibility=internal")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_COMPILER_FLAGS} ${COMPILER_WARN_FLAGS} -D_GNU_SOURCE -fvisibility=internal")
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CXX_COMPILER_FLAGS} -O0 -g -DENABLE_DEBUG")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CXX_COMPILER_FLAGS} -O3")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${CXX_COMPILER_FLAGS} -O2 -g")
@@ -162,6 +195,9 @@ if (NOT ${CMAKE_CXX_COMPILER_ID} STREQUAL "Intel")
 endif()
 
 set(COMMON_CMAKE_DIR ${PROJECT_SOURCE_DIR}/cmake)
+
+define_compute_backend()
+
 if (COMPUTE_BACKEND)
     precheck_compute_backend()
     message(STATUS "COMPUTE_BACKEND: ${COMPUTE_BACKEND}")
@@ -170,6 +206,9 @@ if (COMPUTE_BACKEND)
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCCL_ENABLE_OFI_HMEM=1")
         message(STATUS "Enable OFI HMEM support for compute backend ${COMPUTE_BACKEND}")
     endif()
+    if (${COMPUTE_BACKEND} STREQUAL "dpcpp" AND ${CMAKE_CXX_COMPILER} MATCHES ".*icpx")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl")
+    endif()
 endif()
 
 if (ENABLE_OFI_OOT_PROV)
@@ -182,6 +221,16 @@ if (ENABLE_ITT)
     message(STATUS "Enable ITT profiling support")
 endif()
 
+if (ENABLE_PMIX)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCCL_ENABLE_PMIX=1")
+    message(STATUS "Enable PMIX support")
+endif()
+
+if (ENABLE_DRM)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCCL_ENABLE_DRM=1")
+    message(STATUS "Enable DRM support")
+endif()
+
 if (ENABLE_STUB_BACKEND)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCCL_ENABLE_STUB_BACKEND=1")
     message(STATUS "Enable stub backend")
@@ -225,8 +274,8 @@ enable_testing()
 
 set(EXTERNAL_LIBS "")
 
-set(EXAMPLES_INC_DIRS ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/examples/include ${MPI_INCLUDE_DIR} ${LEVEL_ZERO_INCLUDE_DIR})
-set(EXAMPLES_LIB_DIRS ${MPI_LIB_DIR} ${LIBFABRIC_LIB_DIR})
+set(EXAMPLES_INC_DIRS ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/examples/include ${MPI_INCLUDE_DIR})
+set(EXAMPLES_LIB_DIRS ${MPI_LIB_DIR})
 
 # allow `deprecated`
 set(CMAKE_CLANG_FLAGS "${CMAKE_CLANG_FLAGS}")
@@ -253,8 +302,8 @@ file(GLOB spv_kernels "${PROJECT_SOURCE_DIR}/src/kernels/kernels.spv")
 endif()
 
 set(CCL_MAJOR_VERSION     "2021")
-set(CCL_MINOR_VERSION     "7")
-set(CCL_UPDATE_VERSION    "1")
+set(CCL_MINOR_VERSION     "8")
+set(CCL_UPDATE_VERSION    "0")
 set(CCL_PRODUCT_STATUS    "Gold")
 string(TIMESTAMP CCL_PRODUCT_BUILD_DATE "%Y-%m-%dT %H:%M:%SZ")
 get_vcs_properties("git")

INSTALL.md

@@ -55,7 +55,7 @@ If your CXX compiler requires SYCL, it is possible to specify it (DPC++ is suppo
 Modify `cmake` command as follows:
 
 ```
-cmake .. -DCMAKE_C_COMPILER=your_c_compiler -DCMAKE_CXX_COMPILER=dpcpp -DCOMPUTE_BACKEND=dpcpp
+cmake .. -DCMAKE_C_COMPILER=your_c_compiler -DCMAKE_CXX_COMPILER=icpx -DCOMPUTE_BACKEND=dpcpp
 ```
 
 ## Specify the build type

cmake/FindIntelSYCL_level_zero.cmake

@@ -40,7 +40,7 @@ get_filename_component(INTEL_SYCL_BINARY_DIR ${CMAKE_CXX_COMPILER} PATH)
 
 # Try to find Intel SYCL version.hpp header
 find_path(INTEL_SYCL_INCLUDE_DIRS
-    NAMES CL/sycl/version.hpp
+    NAMES CL/sycl/version.hpp sycl/version.hpp
     PATHS
       ${sycl_root_hints}
       "${INTEL_SYCL_BINARY_DIR}/.."

cmake/helpers.cmake

@@ -22,7 +22,7 @@ function(set_lp_env)
     else()
         set(CCL_BF16_COMPILER OFF)
     endif()
-    message(STATUS "BF16 compiler: ${CCL_BF16_COMPILER}")
+    message(STATUS "BF16 AVX512F compiler: ${CCL_BF16_COMPILER}")
 
     execute_process(COMMAND ld -v
             OUTPUT_VARIABLE BINUTILS_VERSION_RAW
@@ -55,7 +55,7 @@ function(set_lp_env)
         message(STATUS "BF16 target attributes: ${CCL_BF16_TARGET_ATTRIBUTES}")
     endif()
 
-    option(CCL_BF16_GPU_TRUNCATE "Truncate BF16 in GPU operations" ON)
+    option(CCL_BF16_GPU_TRUNCATE "Truncate BF16 in GPU operations" OFF)
     if (CCL_BF16_GPU_TRUNCATE)
         add_definitions(-DCCL_BF16_GPU_TRUNCATE)
     endif()
@@ -204,7 +204,7 @@ function(activate_compute_backend MODULES_PATH COMPUTE_BACKEND)
         # remember current target for `target_link_libraries` in ccl
         set (COMPUTE_BACKEND_TARGET_NAME Intel::SYCL_level_zero)
         set (COMPUTE_BACKEND_TARGET_NAME Intel::SYCL_level_zero PARENT_SCOPE)
-        message ("COMPUTE_BACKEND_TARGET_NAME=${COMPUTE_BACKEND_TARGET_NAME} requested. Using DPC++ provider")
+        message (STATUS "COMPUTE_BACKEND_TARGET_NAME: ${COMPUTE_BACKEND_TARGET_NAME} requested. Using DPC++ provider")
     endif()
 
     # extract target properties
@@ -231,6 +231,18 @@ function(activate_compute_backend MODULES_PATH COMPUTE_BACKEND)
 
 endfunction(activate_compute_backend)
 
+function(define_compute_backend)
+    if (NOT DEFINED COMPUTE_BACKEND)
+        message(STATUS "COMPUTE_BACKEND is not defined")
+        if (${CMAKE_CXX_COMPILER} MATCHES ".*dpcpp")
+            set(COMPUTE_BACKEND "dpcpp" CACHE STRING "compute backend value")
+            message(STATUS "COMPUTE_BACKEND: ${COMPUTE_BACKEND} (set by default)")
+        endif()
+    else()
+        message(STATUS "COMPUTE_BACKEND: ${COMPUTE_BACKEND} (set by user)")
+    endif()
+endfunction(define_compute_backend)
+
 function(set_compute_backend COMMON_CMAKE_DIR)
     activate_compute_backend("${COMMON_CMAKE_DIR}" ${COMPUTE_BACKEND})
 
@@ -257,13 +269,13 @@ function(set_compute_backend COMMON_CMAKE_DIR)
         set(CCL_ENABLE_ZE ON PARENT_SCOPE)
         message(STATUS "Enable CCL Level Zero support")
 
-        execute_process(COMMAND dpcpp -v
-            OUTPUT_VARIABLE DPCPP_VERSION
-            ERROR_VARIABLE DPCPP_VERSION
+        execute_process(COMMAND icpx -v
+            OUTPUT_VARIABLE ICPX_VERSION
+            ERROR_VARIABLE ICPX_VERSION
             OUTPUT_STRIP_TRAILING_WHITESPACE
             ERROR_STRIP_TRAILING_WHITESPACE
         )
-        message(STATUS "DPC++ compiler version:\n" "${DPCPP_VERSION}")
+        message(STATUS "DPC++ compiler version:\n" "${ICPX_VERSION}")
     endif()
 
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPUTE_BACKEND_FLAGS}")

deps/mpi/bin/mpigcc

@@ -115,7 +115,9 @@ fi
 # configure (e.g., determining whehter -lsocket is needee)
 CC="gcc"
 MPICH_VERSION="3.4a2"
-MPIVERSION="2021.7"
+CFLAGS=""
+CPPFLAGS=""
+MPIVERSION="2021.8"
 MPILIBNAME="mpi"                           
 
 

deps/mpi/bin/mpigxx

@@ -113,7 +113,8 @@ fi
 # Default settings for compiler, flags, and libraries
 CXX="g++"
 MPICH_VERSION="3.4a2"
-MPIVERSION="2021.7"
+CXXFLAGS=""
+MPIVERSION="2021.8"
 MPILIBNAME="mpi"
 MPICXXLIBNAME="mpicxx"
 

deps/mpi/bin/mpiicc

@@ -121,7 +121,7 @@ LDFLAGS="-ldl"
 MPILIBNAME="mpi"
 
 # MPIVERSION is the version of the MPICH2 library that mpicc is intended for
-MPIVERSION="2021.7"
+MPIVERSION="2021.8"
 #
 # Internal variables
 # Show is set to echo to cause the compilation command to be echoed instead

deps/mpi/bin/mpiicpc

@@ -122,7 +122,7 @@ MPILIBNAME="mpi"
 MPICXXLIBNAME="mpicxx"
 
 # MPIVERSION is the version of the Intel(R) MPI Library that mpiicpc is intended for
-MPIVERSION="2021.7"
+MPIVERSION="2021.8"
 
 # Internal variables
 # Show is set to echo to cause the compilation command to be echoed instead

deps/mpi/include/mpi.h

@@ -584,8 +584,8 @@ typedef int (MPI_Delete_function) ( MPI_Comm, int, void *, void * );
  * digits for REV, 1 digit for EXT and 2 digits for EXT_NUMBER. So,
  * 2019.0.0b0 will have the numeric version 20190000100.
  */
-#define I_MPI_VERSION "2021.7.1"
-#define I_MPI_NUMVERSION 20210701300
+#define I_MPI_VERSION "2021.8.0"
+#define I_MPI_NUMVERSION 20210800300
 
 /* for the datatype decoders */
 enum MPIR_Combiner_enum {

deps/mpi/licensing/license.txt

@@ -1,33 +1,33 @@
-Intel Simplified Software License (Version August 2021)
+Intel Simplified Software License (Version October 2022)
 
-Use and Redistribution. You may use and redistribute the software (the
-"Software"), without modification, provided the following conditions are met:
+Intel(R) MPI Library: Copyright (C) 2009 Intel Corporation
 
-* Redistributions must reproduce the above copyright notice and the following
-  terms of use in the Software and in the documentation and/or other materials
-  provided with the distribution.
-* Neither the name of Intel nor the names of its suppliers may be used to 
-  endorse or promote products derived from this Software without specific  
-  prior written permission.
-* No reverse engineering, decompilation, or disassembly of this Software is
-  permitted.
+Use and Redistribution. You may use and redistribute the software, which is
+provided in binary form only, (the "Software"), without modification, provided the
+following conditions are met:
+
+* Redistributions must reproduce the above copyright notice and these terms of use
+in the Software and in the documentation and/or other materials provided with
+the distribution.
+* Neither the name of Intel nor the names of its suppliers may be used to endorse
+or promote products derived from this Software without specific prior written
+permission.
+* No reverse engineering, decompilation, or disassembly of the Software is
+permitted, nor any modification or alteration of the Software or its operation
+at any time, including during execution.
 
 No other licenses. Except as provided in the preceding section, Intel grants no
 licenses or other rights by implication, estoppel or otherwise to, patent,
 copyright, trademark, trade name, service mark or other intellectual property
 licenses or rights of Intel.
 
-Third party software. The Software may contain Third Party Software. "Third
-Party Software" is open source software, third party software, or other Intel
-software that may be identified in the Software itself or in the files (if any)
-listed in the "third-party-software.txt" or similarly named text file included
-with the Software. Third Party Software, even if included with the distribution
-of the Software, may be governed by separate license terms, including without
-limitation, open source software license terms, third party software license
-terms, and other Intel software license terms. Those separate license terms
-solely govern your use of the Third Party Software, and nothing in this license
-limits any rights under, or grants rights that supersede, the terms of the
-applicable license terms.
+Third party software. "Third Party Software" means the files (if any) listed in
+the "third-party-software.txt" or other similarly-named text file that may be
+included with the Software. Third Party Software, even if included with the
+distribution of the Software, may be governed by separate license terms, including
+without limitation, third party license terms, open source software notices and
+terms, and/or other Intel software license terms. These separate license terms 
+solely govern Your use of the Third Party Software.
 
 DISCLAIMER. THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
 WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
@@ -63,11 +63,10 @@ Compliance with laws. You agree to comply with all relevant laws and regulations
 governing your use, transfer, import or export (or prohibition thereof) of the
 Software.
 
-Governing law. All disputes will be governed by the laws of the United States of
-America and the State of Delaware without reference to conflict of law
-principles and subject to the exclusive jurisdiction of the state or federal
-courts sitting in the State of Delaware, and each party agrees that it submits
-to the personal jurisdiction and venue of those courts and waives any
-objections. The United Nations Convention on Contracts for the International
-Sale of Goods (1980) is specifically excluded and will not apply to the
-Software.
+Governing law.  All disputes will be governed by the laws of the United States of 
+America and the State of Delaware without reference to conflict of law principles 
+and subject to the exclusive jurisdiction of the state or federal courts sitting 
+in the State of Delaware, and each party agrees that it submits to the personal 
+jurisdiction and venue of those courts and waives any objections. THE UNITED 
+NATIONS CONVENTION ON CONTRACTS FOR THE INTERNATIONAL SALE OF GOODS (1980) IS 
+SPECIFICALLY EXCLUDED AND WILL NOT APPLY TO THE SOFTWARE.

deps/mpi/licensing/third-party-programs.txt

@@ -1,4 +1,4 @@
-Intel(R) MPI Library 2021.7 Third Party Programs File
+Intel(R) MPI Library 2021.8 Third Party Programs File
 
 This file is the "third-party-programs.txt" file specified in the associated 
 Intel end user license agreement for the Intel software you are licensing.