Add NCCL backend

.clang-format

@@ -27,7 +27,6 @@ BreakBeforeBraces: Allman
 # ConstructorInitializerAllOnOneLineOrOnePerLine: false
 BreakConstructorInitializers: BeforeComma
 ConstructorInitializerIndentWidth: 0
-BreakInheritanceList: BeforeComma
 #AllowShortBlocksOnASingleLine: Always
 AllowShortBlocksOnASingleLine: true
 AllowShortCaseLabelsOnASingleLine: false

CMakeLists.txt

@@ -80,6 +80,11 @@ include(oomph_ucx)
 # ---------------------------------------------------------------------
 include(oomph_libfabric)
 
+# ---------------------------------------------------------------------
+# oomph NCCL variant
+# ---------------------------------------------------------------------
+include(oomph_nccl)
+
 # ---------------------------------------------------------------------
 # main src subdir
 # ---------------------------------------------------------------------
@@ -142,6 +147,7 @@ install(
         ${CMAKE_CURRENT_BINARY_DIR}/oomphConfig.cmake
         ${CMAKE_CURRENT_BINARY_DIR}/oomphConfigVersion.cmake
         ${CMAKE_CURRENT_LIST_DIR}/cmake/FindLibfabric.cmake
+        ${CMAKE_CURRENT_LIST_DIR}/cmake/FindNCCL.cmake
         ${CMAKE_CURRENT_LIST_DIR}/cmake/FindUCX.cmake
         ${CMAKE_CURRENT_LIST_DIR}/cmake/FindPMIx.cmake
     DESTINATION

README.md

@@ -4,9 +4,9 @@
 # OOMPH
 
 **Oomph** is a library for enabling high performance point-to-point, asynchronous communication over
-different fabrics.  It leverages the ubiquitos MPI library as well as UCX and Libfabric.  Both
-device and host memory are supported. Under the hood it uses
-[hwmalloc](https://github.com/ghex-org/hwmalloc) for memory registration.
+different fabrics. It leverages the ubiquitous MPI library as well as UCX and Libfabric. Both
+device and host memory are supported. A subset of functionality is also supported with NCCL. Under
+the hood it uses [hwmalloc](https://github.com/ghex-org/hwmalloc) for memory registration.
 
 **selling points**
 - lightweight, fast

cmake/FindNCCL.cmake

@@ -0,0 +1,78 @@
+# From https://github.com/pytorch/gloo/blob/main/cmake/Modules/Findnccl.cmake.
+
+# Try to find NCCL
+#
+# The following variables are optionally searched for defaults
+#  NCCL_ROOT_DIR: Base directory where all NCCL components are found
+#  NCCL_INCLUDE_DIR: Directory where NCCL header is found
+#  NCCL_LIB_DIR: Directory where NCCL library is found
+#
+# The following are set after configuration is done:
+#  NCCL_FOUND
+#  NCCL_INCLUDE_DIRS
+#  NCCL_LIBRARIES
+#
+# The path hints include CUDA_TOOLKIT_ROOT_DIR seeing as some folks
+# install NCCL in the same location as the CUDA toolkit.
+# See https://github.com/caffe2/caffe2/issues/1601
+
+set(NCCL_ROOT_DIR $ENV{NCCL_ROOT_DIR} CACHE PATH "Folder contains NVIDIA NCCL")
+
+find_path(NCCL_INCLUDE_DIR
+  NAMES nccl.h
+  HINTS
+  ${NCCL_INCLUDE_DIR}
+  ${NCCL_ROOT_DIR}
+  ${NCCL_ROOT_DIR}/include
+  ${CUDA_TOOLKIT_ROOT_DIR}/include)
+
+if(DEFINED ENV{USE_STATIC_NCCL} AND NOT "$ENV{USE_STATIC_NCCL}" STREQUAL "")
+  message(STATUS "USE_STATIC_NCCL detected. Linking against static NCCL library")
+  set(_use_static_nccl ON)
+  set(NCCL_LIBNAME "libnccl_static.a")
+else()
+  set(_use_static_nccl OFF)
+  set(NCCL_LIBNAME "nccl")
+endif()
+
+find_library(NCCL_LIBRARY
+  NAMES ${NCCL_LIBNAME}
+  HINTS
+  ${NCCL_LIB_DIR}
+  ${NCCL_ROOT_DIR}
+  ${NCCL_ROOT_DIR}/lib
+  ${NCCL_ROOT_DIR}/lib/x86_64-linux-gnu
+  ${NCCL_ROOT_DIR}/lib64
+  ${CUDA_TOOLKIT_ROOT_DIR}/lib64)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NCCL DEFAULT_MSG NCCL_INCLUDE_DIR NCCL_LIBRARY)
+
+if (NCCL_FOUND)
+  set(NCCL_HEADER_FILE "${NCCL_INCLUDE_DIR}/nccl.h")
+  message(STATUS "Determining NCCL version from the header file: ${NCCL_HEADER_FILE}")
+  file (STRINGS ${NCCL_HEADER_FILE} NCCL_MAJOR_VERSION_DEFINED
+        REGEX "^[ \t]*#define[ \t]+NCCL_MAJOR[ \t]+[0-9]+.*$" LIMIT_COUNT 1)
+  if (NCCL_MAJOR_VERSION_DEFINED)
+    string (REGEX REPLACE "^[ \t]*#define[ \t]+NCCL_MAJOR[ \t]+" ""
+            NCCL_MAJOR_VERSION ${NCCL_MAJOR_VERSION_DEFINED})
+    message(STATUS "NCCL_MAJOR_VERSION: ${NCCL_MAJOR_VERSION}")
+  endif()
+  set(NCCL_INCLUDE_DIRS ${NCCL_INCLUDE_DIR})
+  set(NCCL_LIBRARIES ${NCCL_LIBRARY})
+  message(STATUS "Found NCCL (include: ${NCCL_INCLUDE_DIRS}, library: ${NCCL_LIBRARIES})")
+  mark_as_advanced(NCCL_ROOT_DIR NCCL_INCLUDE_DIRS NCCL_LIBRARIES)
+
+  if(NOT TARGET NCCL::nccl AND NCCL_FOUND)
+    if(_use_static_nccl)
+      add_library(NCCL::nccl STATIC IMPORTED)
+    else()
+      add_library(NCCL::nccl SHARED IMPORTED)
+    endif()
+    set_target_properties(NCCL::nccl PROPERTIES
+      IMPORTED_LOCATION ${NCCL_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${NCCL_INCLUDE_DIRS}
+    )
+  endif()
+endif()
+

cmake/oomph_nccl.cmake

@@ -0,0 +1,19 @@
+# set all NCCL related options and values
+
+#------------------------------------------------------------------------------
+# Enable NCCL support
+#------------------------------------------------------------------------------
+set(OOMPH_WITH_NCCL OFF CACHE BOOL "Build with NCCL backend")
+
+if (OOMPH_WITH_NCCL)
+    find_package(CUDAToolkit REQUIRED)
+    find_package(NCCL REQUIRED)
+    add_library(oomph_nccl SHARED)
+    add_library(oomph::nccl ALIAS oomph_nccl)
+    oomph_shared_lib_options(oomph_nccl)
+    target_link_libraries(oomph_nccl PUBLIC NCCL::nccl CUDA::cudart)
+    install(TARGETS oomph_nccl
+        EXPORT oomph-targets
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endif()