Jb/libfabric fixes

.gitmodules

@@ -2,7 +2,3 @@
 	path = ext/hwmalloc
     url = https://github.com/ghex-org/hwmalloc.git
     branch = master
-[submodule "ext/googletest"]
-	path = ext/googletest
-	url = https://github.com/google/googletest.git
-    branch = main

CMakeLists.txt

@@ -34,6 +34,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 option(OOMPH_GIT_SUBMODULE "Check submodules during build" ON)
 option(OOMPH_USE_BUNDLED_LIBS "Use bundled 3rd party libraries" OFF)
 include(oomph_external_dependencies)
+include(ExternalProject)
 
 # ---------------------------------------------------------------------
 # Define main oomph library
@@ -94,6 +95,8 @@ add_subdirectory(bindings)
 # ---------------------------------------------------------------------
 set(OOMPH_WITH_TESTING OFF CACHE BOOL "True if tests shall be built")
 if (OOMPH_WITH_TESTING)
+    set(CTEST_TEST_TIMEOUT 30)
+    include(CTest)
     enable_testing()
     add_subdirectory(test)
 endif()

benchmarks/CMakeLists.txt

@@ -22,6 +22,7 @@ function(make_benchmark t_ lib)
     add_executable(${t} ${t_}_mt.cpp)
     oomph_target_compile_options(${t})
     target_link_libraries(${t} PRIVATE oomph_${lib})
+    target_compile_definitions(${t} PRIVATE OOMPH_BENCHMARKS_${lib})
 endfunction()
 
 function(make_benchmark_mt t_ lib)
@@ -30,6 +31,7 @@ function(make_benchmark_mt t_ lib)
     add_executable(${t} ${t_}_mt.cpp)
     oomph_target_compile_options(${t})
     target_compile_definitions(${t} PRIVATE OOMPH_BENCHMARKS_MT)
+    target_compile_definitions(${t} PRIVATE OOMPH_BENCHMARKS_${lib})
     target_link_libraries(${t} PRIVATE oomph_${lib})
     target_link_libraries(${t} PRIVATE OpenMP::OpenMP_CXX)
 endfunction()
@@ -104,6 +106,7 @@ if (OOMPH_WITH_LIBFABRIC)
             make_benchmark_mt(${t} libfabric)
         endif()
     endforeach()
+    add_subdirectory(scripts)
 endif()
 else()
     message("warning: benchmarks cannot be built unless barrier is enabled")

benchmarks/mpi_environment.hpp

@@ -12,6 +12,25 @@
 #include <mpi.h>
 #include <iostream>
 
+#ifdef OOMPH_BENCHMARKS_mpi
+#define TRANSPORT_STRING "mpi"
+#define PROGRESS_STRING  "unspecified"
+#define ENDPOINT_STRING  "unspecified"
+#endif
+
+#ifdef OOMPH_BENCHMARKS_ucx
+#define TRANSPORT_STRING "ucx"
+#define PROGRESS_STRING  "unspecified"
+#define ENDPOINT_STRING  "unspecified"
+#endif
+
+#ifdef OOMPH_BENCHMARKS_libfabric
+#include "../src/libfabric/controller.hpp"
+#define TRANSPORT_STRING "libfabric"
+#define PROGRESS_STRING  LIBFABRIC_PROGRESS_STRING
+#define ENDPOINT_STRING  LIBFABRIC_ENDPOINT_STRING
+#endif
+
 namespace oomph
 {
 struct mpi_environment

benchmarks/mpi_p2p_bi_avail_mt.cpp

@@ -23,6 +23,13 @@
 #include <omp.h>
 #endif /* OOMPH_BENCHMARKS_MT */
 
+#ifdef USE_TESTANY
+const char* syncmode = "testany";
+#else
+const char* syncmode = "test";
+#endif
+const char* waitmode = "avail";
+
 int
 main(int argc, char* argv[])
 {
@@ -209,8 +216,26 @@ main(int argc, char* argv[])
     MPI_Barrier(MPI_COMM_WORLD);
     if (rank == 1)
     {
+        double elapsed = t1.toc();
         t1.vtoc();
         t1.vtoc("final ", (double)niter * size * buff_size);
+        double bw = ((double)(niter * size) * buff_size) / elapsed;
+        // clang-format off
+        std::cout << "time:       " << elapsed/1000000 << "s\n";
+        std::cout << "final MB/s: " << bw << "\n";
+        std::cout << "CSVData"
+                  << ", niter, " << niter
+                  << ", buff_size, " << buff_size
+                  << ", inflight, " << inflight
+                  << ", num_threads, " << cmd_args.num_threads
+                  << ", syncmode, " << syncmode
+                  << ", waitmode, " << waitmode
+                  << ", transport, " << "Native-MPI"
+                  << ", BW MB/s, " << bw
+                  << ", progress, " << "unspecified"
+                  << ", endpoint, " << "unspecified"
+                  << "\n";
+        // clang-format on
     }
 
     return 0;

benchmarks/mpi_p2p_bi_wait_mt.cpp

@@ -23,6 +23,13 @@
 #include <omp.h>
 #endif /* OOMPH_BENCHMARKS_MT */
 
+#ifdef USE_WAITALL
+const char* syncmode = "waitall";
+#else
+const char* syncmode = "wait";
+#endif
+const char* waitmode = "wait";
+
 int
 main(int argc, char* argv[])
 {
@@ -155,8 +162,26 @@ main(int argc, char* argv[])
     MPI_Barrier(MPI_COMM_WORLD);
     if (rank == 1)
     {
+        double elapsed = t1.toc();
         t1.vtoc();
         t1.vtoc("final ", (double)niter * size * buff_size);
+        double bw = ((double)(niter * size) * buff_size) / elapsed;
+        // clang-format off
+        std::cout << "time:       " << elapsed/1000000 << "s\n";
+        std::cout << "final MB/s: " << bw << "\n";
+        std::cout << "CSVData"
+                  << ", niter, " << niter
+                  << ", buff_size, " << buff_size
+                  << ", inflight, " << inflight
+                  << ", num_threads, " << cmd_args.num_threads
+                  << ", syncmode, " << syncmode
+                  << ", waitmode, " << waitmode
+                  << ", transport, " << "Native-MPI"
+                  << ", BW MB/s, " << bw
+                  << ", progress, " << "unspecified"
+                  << ", endpoint, " << "unspecified"
+                  << "\n";
+        // clang-format on
     }
 
     return 0;

benchmarks/scripts/CMakeLists.txt

@@ -0,0 +1,68 @@
+#-------------------------------------------------------
+# Slurm job launching script generator
+#
+# We would like to generate a script which can be used to generate batch jobs
+# to submit benchmarks to slurm using many combinations of settings.
+#
+# We add a custom command which takes our template script and
+# expands out all the variables we need to pass into it.
+# Unfortunately, due to the way cmake works, some variables are only known
+# at build time, and not at cmake configure time. Using a custom command which
+# calls cmake to run our script at build time, allows us to pass variables
+# into the final script which is placed in our build dir.
+#
+# Note that we generate these scripts in the build dir instead
+# of the install dir as they are intended for development testing.
+# A version could be supported for installation later.
+#-------------------------------------------------------
+
+set(OOMPH_BENCHMARK_SCRIPTS_PATH "${PROJECT_BINARY_DIR}/scripts" CACHE PATH "Directory to place batch scripts in")
+mark_as_advanced(OOMPH_BENCHMARK_SCRIPTS_PATH)
+
+# Make sure scripts dir exists
+execute_process(COMMAND "${CMAKE_COMMAND}" -E make_directory "${OOMPH_BENCHMARK_SCRIPTS_PATH}")
+
+# The MPI backend is always enabled and uses no extension on the benchmarks
+# it will be added automatically by the script runner
+set(BENCHMARK_SUFFIXES)
+
+if (OOMPH_WITH_LIBFABRIC)
+    list(APPEND BENCHMARK_SUFFIXES _libfabric)
+endif()
+if (OOMPH_WITH_UCX)
+    list(APPEND BENCHMARK_SUFFIXES _ucx)
+endif()
+
+#--------------------------------------------------
+# Slurm script generator for benchmarks
+#--------------------------------------------------
+set(SCRIPTS "generate-oomph.py")
+
+foreach(script ${SCRIPTS})
+  ADD_CUSTOM_COMMAND(
+    DEPENDS   "${CMAKE_CURRENT_SOURCE_DIR}/${script}"
+    COMMAND   "${CMAKE_COMMAND}"
+    ARGS
+              # needed by the benchmark script
+              -DBIN_DIR=${PROJECT_BINARY_DIR}/benchmarks
+              -DRUN_DIR=${OOMPH_BENCHMARK_SCRIPTS_PATH}
+              -DMPIEXEC="${MPIEXEC}"
+
+              # needed by the copy script
+              -DSCRIPT_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}"
+              -DSCRIPT_NAME=${script}
+              -DSCRIPT_DEST_DIR="${OOMPH_BENCHMARK_SCRIPTS_PATH}"
+
+              # quoted as it might be a list
+              "-DBENCHMARK_SUFFIXES=${BENCHMARK_SUFFIXES}"
+              -DJOB_OPTIONS1="${SLURM_JOB_OPTIONS1}"
+              -P "${CMAKE_CURRENT_SOURCE_DIR}/copy_script.cmake"
+
+    OUTPUT    "${OOMPH_BENCHMARK_SCRIPTS_PATH}/${script}"
+    VERBATIM
+  )
+
+  add_custom_target(script-${script}
+    DEPENDS "${OOMPH_BENCHMARK_SCRIPTS_PATH}/${script}"
+  )
+endforeach(script)

benchmarks/scripts/copy_script.cmake

@@ -0,0 +1,22 @@
+FILE(TO_NATIVE_PATH "${SCRIPT_SOURCE_DIR}/${SCRIPT_NAME}" INFILE)
+FILE(TO_NATIVE_PATH "${SCRIPT_DEST_DIR}/${SCRIPT_NAME}" OUTFILE)
+
+STRING(REPLACE "\"" "" FILE1 "${INFILE}")
+STRING(REPLACE "\"" "" FILE2 "${OUTFILE}")
+
+#
+# when debugging, this dumps all vars out
+#
+if (DEBUG_THIS_SCRIPT)
+    get_cmake_property(_variableNames VARIABLES)
+    list (SORT _variableNames)
+    foreach (_variableName ${_variableNames})
+        message(STATUS "${_variableName}=${${_variableName}}")
+    endforeach()
+endif()
+
+configure_file(
+  "${FILE1}"
+  "${FILE2}"
+  @ONLY
+)