Merge pull request #1307 from LLNL/bugfix/chin23/no-raja-umpire

Fixes for builds with no raja or umpire

src/serac/infrastructure/CMakeLists.txt

@@ -21,6 +21,7 @@ set(infrastructure_headers
     initialize.hpp
     input.hpp
     logger.hpp
+    memory.hpp
     mpi_fstream.hpp
     output.hpp
     profiling.hpp
@@ -41,9 +42,12 @@ set(infrastructure_sources
     terminator.cpp
     )
 
-set(infrastructure_depends axom::inlet axom::fmt axom::cli11 mfem ${serac_device_depends})
+set(infrastructure_depends axom::inlet axom::fmt axom::cli11 camp mfem ${serac_device_depends})
 blt_list_append(TO infrastructure_depends ELEMENTS tribol IF TRIBOL_FOUND)
 blt_list_append(TO infrastructure_depends ELEMENTS caliper adiak::adiak IF SERAC_ENABLE_PROFILING)
+# TODO (EBC): mfem's openmp dependency isn't being picked up on ubuntu + clang 16.0.6.  
+#             This adds it in manually for now, but should be investigated more fully.
+blt_list_append(TO infrastructure_depends ELEMENTS blt::openmp IF SERAC_ENABLE_OPENMP)
 list(APPEND infrastructure_depends blt::mpi)
 
 blt_add_library(

src/serac/infrastructure/accelerator.hpp

@@ -55,6 +55,7 @@
 #include "axom/core.hpp"
 
 #include "serac/infrastructure/logger.hpp"
+#include "serac/infrastructure/memory.hpp"
 #include "serac/infrastructure/profiling.hpp"
 
 /**
@@ -121,7 +122,7 @@ void zero_out(axom::Array<T, dim, space>& arr)
 
 /// @brief set the contents of an array to zero, byte-wise
 template <typename T, int dim>
-void zero_out(axom::ArrayView<T, dim, axom::MemorySpace::Host>& arr)
+void zero_out(axom::ArrayView<T, dim, detail::host_memory_space>& arr)
 {
   std::memset(arr.data(), 0, static_cast<std::size_t>(arr.size()) * sizeof(T));
 }

src/serac/infrastructure/debug_print.hpp

@@ -13,6 +13,9 @@
 #include <iomanip>
 #include <vector>
 
+#include "serac/infrastructure/memory.hpp"
+#include "serac/numerics/functional/element_restriction.hpp"
+
 /**
  * @brief write an array of values out to file, in a space-separated format
  * @tparam T the type of each value in the array
@@ -68,7 +71,7 @@ std::ostream& operator<<(std::ostream& out, DoF dof)
  * @param filename the name of the output file
  */
 template <typename T>
-void write_to_file(axom::Array<T, 2, axom::MemorySpace::Host> arr, std::string filename)
+void write_to_file(axom::Array<T, 2, serac::detail::host_memory_space> arr, std::string filename)
 {
   std::ofstream outfile(filename);
 
@@ -91,7 +94,7 @@ void write_to_file(axom::Array<T, 2, axom::MemorySpace::Host> arr, std::string f
  * @param filename the name of the output file
  */
 template <typename T>
-void write_to_file(axom::Array<T, 3, axom::MemorySpace::Host> arr, std::string filename)
+void write_to_file(axom::Array<T, 3, serac::detail::host_memory_space> arr, std::string filename)
 {
   std::ofstream outfile(filename);
 

src/serac/infrastructure/memory.hpp

@@ -0,0 +1,34 @@
+// Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and
+// other Serac Project Developers. See the top-level LICENSE file for
+// details.
+//
+// SPDX-License-Identifier: (BSD-3-Clause)
+
+/**
+ * @file memory.hpp
+ *
+ * @brief This file defines the host memory space
+ */
+
+#pragma once
+
+#include "axom/core.hpp"
+
+#include "serac/serac_config.hpp"
+
+namespace serac {
+
+namespace detail {
+
+/**
+ * @brief Sets the axom memory space based on whether or not Umpire is being used
+ */
+#ifdef SERAC_USE_UMPIRE
+constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Host;
+#else
+constexpr axom::MemorySpace host_memory_space = axom::MemorySpace::Dynamic;
+#endif
+
+}  // namespace detail
+
+}  // namespace serac

src/serac/numerics/functional/domain_integral_kernels.hpp

@@ -10,7 +10,9 @@
 #include "serac/numerics/functional/quadrature_data.hpp"
 #include "serac/numerics/functional/function_signature.hpp"
 #include "serac/numerics/functional/differentiate_wrt.hpp"
+#ifdef SERAC_USE_RAJA
 #include "RAJA/RAJA.hpp"
+#endif
 
 #include <array>
 #include <cstdint>

src/serac/numerics/functional/element_restriction.cpp

@@ -216,7 +216,8 @@ std::vector<Array2D<int> > geom_local_face_dofs(int p)
   return output;
 }
 
-axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementRestriction(const serac::fes_t* fes, mfem::Geometry::Type geom)
+axom::Array<DoF, 2, serac::detail::host_memory_space> GetElementRestriction(const serac::fes_t* fes,
+                                                                            mfem::Geometry::Type geom)
 {
   std::vector<DoF> elem_dofs{};
   mfem::Mesh* mesh = fes->GetMesh();
@@ -269,17 +270,17 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementRestriction(const serac::
   }
 
   if (n == 0) {
-    return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
+    return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
   } else {
     uint64_t dofs_per_elem = elem_dofs.size() / n;
-    axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_elem);
+    axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_elem);
     std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem);
     return output;
   }
 }
 
-axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom,
-                                                            const std::vector<int>& mfem_elem_ids)
+axom::Array<DoF, 2, serac::detail::host_memory_space> GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom,
+                                                                     const std::vector<int>& mfem_elem_ids)
 
 {
   std::vector<DoF> elem_dofs{};
@@ -335,17 +336,17 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t*
   }
 
   if (n == 0) {
-    return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
+    return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
   } else {
     uint64_t dofs_per_elem = elem_dofs.size() / n;
-    axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_elem);
+    axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_elem);
     std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem);
     return output;
   }
 }
 
-axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
-                                                         FaceType type)
+axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
+                                                                  mfem::Geometry::Type face_geom, FaceType type)
 {
   std::vector<DoF> face_dofs;
   mfem::Mesh* mesh = fes->GetMesh();
@@ -450,17 +451,18 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes
   delete face_to_elem;
 
   if (n == 0) {
-    return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
+    return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
   } else {
     uint64_t dofs_per_face = face_dofs.size() / n;
-    axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_face);
+    axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_face);
     std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face);
     return output;
   }
 }
 
-axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
-                                                         const std::vector<int>& mfem_face_ids)
+axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
+                                                                  mfem::Geometry::Type face_geom,
+                                                                  const std::vector<int>& mfem_face_ids)
 {
   std::vector<DoF> face_dofs;
   mfem::Mesh* mesh = fes->GetMesh();
@@ -617,10 +619,10 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes
   delete face_to_elem;
 
   if (n == 0) {
-    return axom::Array<DoF, 2, axom::MemorySpace::Host>{};
+    return axom::Array<DoF, 2, serac::detail::host_memory_space>{};
   } else {
     uint64_t dofs_per_face = face_dofs.size() / n;
-    axom::Array<DoF, 2, axom::MemorySpace::Host> output(n, dofs_per_face);
+    axom::Array<DoF, 2, serac::detail::host_memory_space> output(n, dofs_per_face);
     std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face);
     return output;
   }

src/serac/numerics/functional/element_restriction.hpp

@@ -7,6 +7,7 @@
 #include "geometry.hpp"
 #include "domain.hpp"
 
+#include "serac/infrastructure/memory.hpp"
 #include "serac/numerics/functional/typedefs.hpp"
 
 inline bool isH1(const mfem::FiniteElementSpace& fes)
@@ -197,7 +198,7 @@ struct ElementRestriction {
   uint64_t nodes_per_elem;
 
   /// a 2D array (num_elements-by-nodes_per_elem) holding the dof info extracted from the finite element space
-  axom::Array<DoF, 2, axom::MemorySpace::Host> dof_info;
+  axom::Array<DoF, 2, serac::detail::host_memory_space> dof_info;
 
   /// whether the underlying dofs are arranged "byNodes" or "byVDim"
   mfem::Ordering::Type ordering;
@@ -242,7 +243,8 @@ struct BlockElementRestriction {
  * @param fes the finite element space containing the dof information
  * @param geom the kind of element geometry
  */
-axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t* fes, mfem::Geometry::Type geom);
+axom::Array<DoF, 2, serac::detail::host_memory_space> GetElementDofs(const serac::fes_t* fes,
+                                                                     mfem::Geometry::Type geom);
 
 /**
  * @brief Get the list of dofs for each face element (of the specified geometry) from the fes_t
@@ -251,9 +253,10 @@ axom::Array<DoF, 2, axom::MemorySpace::Host> GetElementDofs(const serac::fes_t*
  * @param geom the kind of element geometry
  * @param type whether the face is of interior or boundary type
  */
-axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
-                                                         FaceType type);
+axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
+                                                                  mfem::Geometry::Type face_geom, FaceType type);
 
 /// @overload
-axom::Array<DoF, 2, axom::MemorySpace::Host> GetFaceDofs(const serac::fes_t* fes, mfem::Geometry::Type face_geom,
-                                                         const std::vector<int>& mfem_face_ids);
+axom::Array<DoF, 2, serac::detail::host_memory_space> GetFaceDofs(const serac::fes_t* fes,
+                                                                  mfem::Geometry::Type face_geom,
+                                                                  const std::vector<int>& mfem_face_ids);

src/serac/numerics/functional/tests/dg_restriction_operators.cpp

@@ -229,7 +229,7 @@ mfem::Mesh generate_permuted_mesh(mfem::Geometry::Type geom, int i)
   return {};
 }
 
-std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, axom::MemorySpace::Host> arr)
+std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, serac::detail::host_memory_space> arr)
 {
   for (int i = 0; i < arr.shape()[0]; i++) {
     for (int j = 0; j < arr.shape()[1]; j++) {

src/serac/numerics/functional/tests/element_restriction_tests.cpp

@@ -5,7 +5,7 @@
 
 using namespace serac;
 
-std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, axom::MemorySpace::Host> arr)
+std::ostream& operator<<(std::ostream& out, axom::Array<DoF, 2, serac::detail::host_memory_space> arr)
 {
   for (int i = 0; i < arr.shape()[0]; i++) {
     for (int j = 0; j < arr.shape()[1]; j++) {

src/serac/numerics/functional/tests/functional_basic_dg.cpp

@@ -39,7 +39,8 @@ void L2_test(std::string meshfile)
   mfem::ParFiniteElementSpace fespace(mesh.get(), &fec, dim, serac::ordering);
 
   mfem::Vector U(fespace.TrueVSize());
-  U.Randomize();
+  int seed = 1;
+  U.Randomize(seed);
 
   // Construct the new functional object using the specified test and trial spaces
   Functional<test_space(trial_space)> residual(&fespace, {&fespace});
@@ -96,8 +97,8 @@ void L2_qoi_test(std::string meshfile)
   auto fec = mfem::L2_FECollection(p, dim, mfem::BasisType::GaussLobatto);
   mfem::ParFiniteElementSpace fespace(mesh.get(), &fec, dim, serac::ordering);
 
-  int seed = 0;
   mfem::HypreParVector U = *fespace.NewTrueDofVector();
+  int seed = 2;
   U.Randomize(seed);
 
   // Construct the new functional object using the specified test and trial spaces
@@ -162,10 +163,12 @@ void L2_scalar_valued_test(std::string meshfile)
   mfem::ParFiniteElementSpace fespace_1(mesh.get(), &H1fec, dim, serac::ordering);
 
   mfem::Vector U0(fespace_0.TrueVSize());
-  U0.Randomize();
+  int seed = 3;
+  U0.Randomize(seed);
 
   mfem::Vector U1(fespace_1.TrueVSize());
-  U1.Randomize();
+  seed = 4;
+  U1.Randomize(seed);
 
   // Construct the new functional object using the specified test and trial spaces
   Functional<test_space(trial_space_0, trial_space_1)> residual(&fespace_0, {&fespace_0, &fespace_1});

src/serac/numerics/functional/tests/functional_basic_h1_vector.cpp

@@ -97,7 +97,8 @@ void weird_mixed_test(std::unique_ptr<mfem::ParMesh>& mesh)
   mfem::Vector U(trial_fes->TrueVSize());
 
   Functional<test_space(trial_space), exec_space> residual(test_fes.get(), {trial_fes.get()});
-  U.Randomize();
+  int seed = 5;
+  U.Randomize(seed);
 
   // note: this is not really an elasticity problem, it's testing source and flux
   // terms that have the appropriate shapes to ensure that all the differentiation
@@ -122,7 +123,8 @@ void elasticity_test(std::unique_ptr<mfem::ParMesh>& mesh)
   auto [test_fes, test_col] = generateParFiniteElementSpace<test_space>(mesh.get());
 
   mfem::Vector U(trial_fes->TrueVSize());
-  U.Randomize();
+  int seed = 6;
+  U.Randomize(seed);
 
   Functional<test_space(trial_space), exec_space> residual(test_fes.get(), {trial_fes.get()});
 

src/serac/numerics/functional/tests/functional_basic_hcurl.cpp

@@ -40,7 +40,8 @@ void hcurl_test_2D()
   mfem::ParFiniteElementSpace fespace(mesh.get(), &fec);
 
   mfem::Vector U(fespace.TrueVSize());
-  U.Randomize();
+  int seed = 7;
+  U.Randomize(seed);
 
   // Construct the new functional object using the specified test and trial spaces
   Functional<test_space(trial_space)> residual(&fespace, {&fespace});
@@ -77,7 +78,8 @@ void hcurl_test_3D()
   mfem::ParFiniteElementSpace fespace(mesh.get(), &fec);
 
   mfem::Vector U(fespace.TrueVSize());
-  U.Randomize();
+  int seed = 8;
+  U.Randomize(seed);
 
   // Define the types for the test and trial spaces using the function arguments
   using test_space = Hcurl<p>;

src/serac/numerics/functional/tests/functional_boundary_test.cpp

@@ -88,7 +88,8 @@ void boundary_test(mfem::ParMesh& mesh, H1<p> test, H1<p> trial, Dimension<dim>)
   std::unique_ptr<mfem::HypreParMatrix> J(B.ParallelAssemble());
 
   mfem::Vector U(fespace->TrueVSize());
-  U.Randomize();
+  int seed = 9;
+  U.Randomize(seed);
 
   Functional<test_space(trial_space)> residual(fespace.get(), {fespace.get()});
 
@@ -164,7 +165,8 @@ void boundary_test(mfem::ParMesh& mesh, L2<p> test, L2<p> trial, Dimension<dim>)
   std::unique_ptr<mfem::HypreParMatrix> J(B.ParallelAssemble());
 
   mfem::ParGridFunction u_global(fespace.get());
-  u_global.Randomize();
+  int seed = 1;
+  u_global.Randomize(seed);
   mfem::FunctionCoefficient xfunc([](mfem::Vector x) { return x[0]; });
   u_global.ProjectCoefficient(xfunc);
 

src/serac/numerics/functional/tests/functional_comparison_L2.cpp

@@ -98,7 +98,8 @@ void functional_test(mfem::ParMesh& mesh, L2<p> test, L2<p> trial, Dimension<dim
 
   // Set a random state to evaluate the residual
   mfem::ParGridFunction u_global(fespace.get());
-  u_global.Randomize();
+  int seed = 2;
+  u_global.Randomize(seed);
 
   mfem::Vector U(fespace->TrueVSize());
   u_global.GetTrueDofs(U);

src/serac/numerics/functional/tests/functional_comparisons.cpp

@@ -129,7 +129,8 @@ void functional_test(mfem::ParMesh& mesh, H1<p> test, H1<p> trial, Dimension<dim
 
   // Set a random state to evaluate the residual
   mfem::ParGridFunction u_global(fespace.get());
-  u_global.Randomize();
+  int seed = 6;
+  u_global.Randomize(seed);
 
   mfem::Vector U(fespace->TrueVSize());
   u_global.GetTrueDofs(U);
@@ -254,7 +255,8 @@ void functional_test(mfem::ParMesh& mesh, H1<p, dim> test, H1<p, dim> trial, Dim
   std::unique_ptr<mfem::HypreParVector> F(f.ParallelAssemble());
 
   mfem::ParGridFunction u_global(fespace.get());
-  u_global.Randomize();
+  int seed = 7;
+  u_global.Randomize(seed);
 
   mfem::Vector U(fespace->TrueVSize());
   u_global.GetTrueDofs(U);
@@ -371,7 +373,8 @@ void functional_test(mfem::ParMesh& mesh, Hcurl<p> test, Hcurl<p> trial, Dimensi
   std::unique_ptr<mfem::HypreParVector> F(f.ParallelAssemble());
 
   mfem::ParGridFunction u_global(fespace.get());
-  u_global.Randomize();
+  int seed = 8;
+  u_global.Randomize(seed);
 
   mfem::Vector U(fespace->TrueVSize());
   u_global.GetTrueDofs(U);