From b58a1e6bda812bf61d6809b123604e01e67198f4 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 15 Feb 2024 16:09:49 +0100 Subject: [PATCH 001/170] add Morton & Hilbert class to compute SFC --- include/samurai/hilbert.hpp | 240 ++++++++++++++++++++++++++++++++++++ include/samurai/morton.hpp | 151 +++++++++++++++++++++++ include/sfc.hpp | 40 ++++++ 3 files changed, 431 insertions(+) create mode 100644 include/samurai/hilbert.hpp create mode 100644 include/samurai/morton.hpp create mode 100644 include/sfc.hpp diff --git a/include/samurai/hilbert.hpp b/include/samurai/hilbert.hpp new file mode 100644 index 000000000..b6f5a6407 --- /dev/null +++ b/include/samurai/hilbert.hpp @@ -0,0 +1,240 @@ +#pragma once + +#include +#include "sfc.hpp" + +class Hilbert : public SFC { + + private: + const std::vector>> _2D_STATE = {{{1, 0, 2, 3}, {0, 0, 2, 2}}, + {{0, 3, 2, 1}, {1, 3, 1, 3}}, + {{2, 1, 0, 3}, {3, 1, 3, 1}}, + {{0, 1, 3, 2}, {2, 2, 0, 0}}}; + + const std::vector>> _3D_STATE = {{{1,2,0,6,11,4,5,6,10,4,7,10}, + {0,0,0,2,4,6,4,6,2,2,4,6}}, + {{2,6,9,0,11,4,7,1,3,4,2,3}, + {1,7,3,3,3,5,7,7,5,1,5,1}}, + {{3,0,10,6,0,8,5,6,1,8,11,2}, + {3,1,7,1,5,1,3,5,3,5,7,7}}, + {{2,7,9,11,7,8,3,10,1,8,2,6}, + {2,6,4,0,2,2,0,4,4,6,6,0}}, + {{4,8,1,9,5,0,1,9,10,2,7,10}, + {7,3,1,5,7,7,5,1,1,3,3,5}}, + {{5,8,1,0,9,6,1,4,3,7,5,3}, + {6,4,2,4,0,4,6,0,6,0,2,2}}, + {{3,0,11,9,0,10,11,9,5,2,8,4}, + {4,2,6,6,6,0,2,2,0,4,0,4}}, + {{5,7,11,8,7,6,11,10,9,3,5,4}, + {5,5,5,7,1,3,1,3,7,7,1,3}}}; + + const std::vector>> _3D_STATE_r = {{{1,2,0,11,9,10,3,4,5,7,8,6}, + {0,0,0,3,5,6,3,5,6,5,6,3}}, + {{2,0,1,6,7,8,11,9,10,4,5,3}, + {1,2,4,2,7,2,7,4,4,1,7,1}}, + {{2,0,1,6,7,8,11,9,10,4,5,3}, + {3,6,5,0,3,3,6,6,0,0,5,5}}, + {{3,8,9,0,11,6,5,10,1,2,7,4}, + {2,4,1,1,1,7,2,7,2,4,4,7}}, + {{3,8,9,0,11,6,5,10,1,2,7,4}, + {6,5,3,5,0,5,0,3,3,6,0,6}}, + {{5,7,11,9,0,4,1,6,3,8,2,10}, + {7,7,7,4,2,1,4,2,1,2,1,4}}, + {{5,7,11,9,0,4,1,6,3,8,2,10}, + {5,3,6,6,6,0,5,0,5,3,3,0}}, + {{4,6,10,8,5,0,7,1,9,3,11,2}, + {4,1,2,7,4,4,1,1,7,7,2,2}}}; + + + public: + + /* + * Return the hilbert index from logical coordinate (i,j) or (i,j,k) + * + * Parameters: + * lc : structure containing logical coordinate + * level : used for multiresolution + * Return : + * key : hilbert key + */ + inline SFC_key_t getKey_2D_impl( const Logical_coord & lc, int lvl ) const { + + SFC_key_t la_clef = 0; + + constexpr int dim = 2; + constexpr int twotondim = 1 << dim; + + int ind_bits[lvl][dim]; + int h_digits[lvl]; + + SFC_key_t xyz[dim] = { lc.i, lc.j }; + + // Set ind_bits for current point + for (int ibit = 0; ibit < lvl; ++ibit) { + for( size_t idim=0; idim> ibit) & 1; + } + } + + // Compute Hilbert key bits + int cur_state = 0; + int new_state; + for(int ibit=lvl-1; ibit>-1; --ibit) { + + // Compute s_digit by interleaving bits + int s_digit = 0; + for(size_t idim=0; idim> ibit ) & 1; + } + } + + // Compute Hilbert key bits + int cur_state = 0; + int new_state; + for(int ibit=lvl-1; ibit>-1; --ibit) { + + // Compute s_digit by interleaving bits + int s_digit = 0; + for(size_t idim=0; idim get_logical_from_hilbert_3D(long double hkey, int order, int dim){ + +// int twotondim = 1 << dim; +// std::vector ind_array(dim); +// std::vector h_digits(order); +// std::vector> ind_bits( order, std::vector (dim)); + +// // Compute Hilbert key bits +// for(int ibit=0; ibit((hkey / std::pow(twotondim, ibit))) % twotondim); +// hkey -= h_digits[ibit] * std::pow(twotondim, ibit); +// } + +// // Compute indices bits +// int cur_state = 0, new_state, s_digit = 0; +// for(int ibit=order-1; ibit>-1; --ibit) { + +// // Compute the new s_digit from the state diagram +// new_state = _3D_STATE_r[h_digits[ibit]][0][cur_state]; +// s_digit = _3D_STATE_r[h_digits[ibit]][1][cur_state]; +// cur_state = new_state; + +// // Compute ind_bitd +// for (int idim = 0; idim < dim; ++idim) { +// ind_bits[ibit][idim] = (s_digit >> (dim - 1 - idim)) & 1; +// } +// } + +// // Set indices for current key +// for(int ibit=0; ibit( (hkey / std::pow( twotondim, ibit )) ) % twotondim); +// hkey -= h_digits[ibit] * std::pow(twotondim, ibit); +// } + +// // Compute indices bits +// int cur_state = 0, new_state, s_digit = 0; +// for(int ibit=level-1; ibit>-1; --ibit) { + +// // Compute the new s_digit from the state diagram +// new_state = _HILBERT_3D_STATE_r[h_digits[ibit]][0][cur_state]; +// s_digit = _HILBERT_3D_STATE_r[h_digits[ibit]][1][cur_state]; +// cur_state = new_state; + +// // Compute ind_bitd +// for (int idim = 0; idim < dim; ++idim) { +// ind_bits[ibit][idim] = (s_digit >> (dim - 1 - idim)) & 1; +// } +// } + +// // Set indices for current key +// for(int ibit=0; ibit +#include + +#include "sfc.hpp" + +class Morton : public SFC { + + private: + + /* + * Split bit by 3, version for 2D, used to compute morton index + * + * Parameters: + * logical_a : coordonnées logiques (i ou j) + * + * return : + * x : unsigned 64bits with splited by 3 + */ + inline uint64_t splitBy3_2D( uint32_t logical_a ) const { + + uint64_t x = logical_a & 0xffffffff; + x = (x | x << 16) & 0xffff0000ffff; + x = (x | x << 8) & 0xff00ff00ff00ff; + x = (x | x << 4) & 0xf0f0f0f0f0f0f0f; + x = (x | x << 2) & 0x3333333333333333; + x = (x | x << 1) & 0x5555555555555555; + + return x; + } + + /* + * Split bit by 3, version for 3D, used to compute morton index + * + * Parameters: + * logical_a : coordonnées logiques (i ou j ou k) + * + * return : + * x : unsigned 64bits with splited by 3 + */ + inline uint64_t splitBy3_3D(uint32_t logical_a) const { + + uint64_t x = logical_a & 0x1fffff; + + x = (x | x << 32) & 0x1f00000000ffff; + x = (x | x << 16) & 0x1f0000ff0000ff; + x = (x | x << 8) & 0x100f00f00f00f00f; + x = (x | x << 4) & 0x10c30c30c30c30c3; + x = (x | x << 2) & 0x1249249249249249; + + return x; + + } + + + public: + + /* + * Return the morton index from logical coordinate (i,j) or (i,j,k) + * + * Parameters: + * lc : structure containing logical coordinate + * + * Return : + * key : morton key + */ + inline SFC_key_t getKey_2D_impl( const Logical_coord & lc ) const { + SFC_key_t la_clef = 0; + la_clef |= splitBy3_2D( lc.i ) | splitBy3_2D( lc.j ) << 1; + return la_clef; + } + + inline SFC_key_t getKey_3D_impl( const Logical_coord & lc ) const { + uint64_t la_clef = 0; + la_clef |= splitBy3_3D( lc.i ) | splitBy3_3D( lc.j ) << 1 | splitBy3_3D( lc.k ) << 2; + return la_clef; + } + + /* + * Return the logical coordinate (i,j) or (i,j,k) from the morton index + * + * Parameters: + * clef (in): morton index + * + * Return : + * lc (out): 2D/3D logical coordinates + */ + inline Logical_coord getCoordinates_impl_2D( const SFC_key_t & clef ) const { + + Logical_coord lc {0, 0, 0}; + + // Extract coord i + SFC_key_t keyi = clef >> 0; + keyi &= 0x5555555555555555; + keyi = (keyi ^ (keyi >> 1)) & 0x3333333333333333; + keyi = (keyi ^ (keyi >> 2)) & 0x0f0f0f0f0f0f0f0f; + keyi = (keyi ^ (keyi >> 4)) & 0x00ff00ff00ff00ff; + keyi = (keyi ^ (keyi >> 8)) & 0x0000ffff0000ffff; + keyi = (keyi ^ (keyi >> 16)) & 0x00000000ffffffff; + lc.i = static_cast(keyi); + + // extract coord j + SFC_key_t keyj = clef >> 1; + keyj &= 0x5555555555555555; + keyj = (keyj ^ (keyj >> 1)) & 0x3333333333333333; + keyj = (keyj ^ (keyj >> 2)) & 0x0f0f0f0f0f0f0f0f; + keyj = (keyj ^ (keyj >> 4)) & 0x00ff00ff00ff00ff; + keyj = (keyj ^ (keyj >> 8)) & 0x0000ffff0000ffff; + keyj = (keyj ^ (keyj >> 16)) & 0x00000000ffffffff; + lc.j = static_cast(keyj); + + return lc; + } + + inline Logical_coord getCoordinates_impl_3D( const SFC_key_t & clef ) const { + + Logical_coord lc = {0, 0, 0}; // k=0 + + // Extract coord i + SFC_key_t keyi = clef >> 0; + keyi &= 0x1249249249249249; + keyi = (keyi ^ (keyi >> 2)) & 0x30c30c30c30c30c3; + keyi = (keyi ^ (keyi >> 4)) & 0xf00f00f00f00f00f; + keyi = (keyi ^ (keyi >> 8)) & 0x00ff0000ff0000ff; + keyi = (keyi ^ (keyi >> 16)) & 0x00ff00000000ffff; + keyi = (keyi ^ (keyi >> 32)) & 0x1fffff; + lc.i = keyi; + + SFC_key_t keyj = clef >> 1; + keyj &= 0x1249249249249249; + keyj = (keyj ^ (keyj >> 2)) & 0x30c30c30c30c30c3; + keyj = (keyj ^ (keyj >> 4)) & 0xf00f00f00f00f00f; + keyj = (keyj ^ (keyj >> 8)) & 0x00ff0000ff0000ff; + keyj = (keyj ^ (keyj >> 16)) & 0x00ff00000000ffff; + keyj = (keyj ^ (keyj >> 32)) & 0x1fffff; + lc.j = keyj; + + SFC_key_t keyk = clef >> 2; + keyk &= 0x1249249249249249; + keyk = (keyk ^ (keyk >> 2)) & 0x30c30c30c30c30c3; + keyk = (keyk ^ (keyk >> 4)) & 0xf00f00f00f00f00f; + keyk = (keyk ^ (keyk >> 8)) & 0x00ff0000ff0000ff; + keyk = (keyk ^ (keyk >> 16)) & 0x00ff00000000ffff; + keyk = (keyk ^ (keyk >> 32)) & 0x1fffff; + lc.k = keyk; + + return lc; + } + +}; diff --git a/include/sfc.hpp b/include/sfc.hpp new file mode 100644 index 000000000..0b3500a12 --- /dev/null +++ b/include/sfc.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include + +struct Logical_coord{ + uint32_t i, j, k; +}; + +template +class SFC { + + public: + using SFC_key_t = uint64_t; + + /** + * + * Interface to compute Space Filling Curve 1D index from + * 2D/3D logical coordinates of cell. + * + */ + template + inline SFC_key_t getKey( const Logical_coord & lc, const Args&... kw ) const{ + // return SFC_Flavor::getKey_impl( lc ); + if constexpr ( dim == 2 ) return static_cast(this)->getKey_2D_impl( lc, kw... ); + if constexpr ( dim == 3 ) return static_cast(this)->getKey_3D_impl( lc, kw... ); + }; + + /** + * + * Interface to compute logical coordinate from Space Filling Curve 1D index. + * + */ + template + inline Logical_coord getCoordinates( const SFC_key_t & clef ) const{ + // return SFC_Flavor::getKey_impl( lc ); + if constexpr ( dim == 2 ) return static_cast(this)->getCoordinates_2D_impl( clef ); + if constexpr ( dim == 3 ) return static_cast(this)->getCoordinates_3D_impl( clef ); + }; + +}; \ No newline at end of file From d5140540619fd6d312e76fb6b94b2c981c94ff05 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 15 Feb 2024 16:10:40 +0100 Subject: [PATCH 002/170] add simple timers --- include/samurai/assertLogTrace.hpp | 22 ++++++++++++++++++++++ include/samurai/timers.hpp | 1 + 2 files changed, 23 insertions(+) create mode 100644 include/samurai/assertLogTrace.hpp diff --git a/include/samurai/assertLogTrace.hpp b/include/samurai/assertLogTrace.hpp new file mode 100644 index 000000000..a2f771c20 --- /dev/null +++ b/include/samurai/assertLogTrace.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include + +#define SAMURAI_ASSERT(condition, msg) \ +do { \ + if (! (condition)) { \ + std::cerr << "Assertion failed \nin " << __FILE__ \ + << "\n @line " << __LINE__ << ": " << msg << std::endl; \ + std::terminate(); \ + } \ +} while (false) + +//#ifdef NDEBUG +#define SAMURAI_LOG( msg ) do { std::cerr << "SMR::Log:: " \ + << msg << std::endl; } while (0) + +#define SAMURAI_TRACE( msg ) do { std::cerr << "SMR::Trace[line " << __LINE__ << "] :" \ + << msg << std::endl; } while (0) +//#else +//#define MGS_LOG( msg ) +//#endif diff --git a/include/samurai/timers.hpp b/include/samurai/timers.hpp index e3ab002a5..90b764e5a 100644 --- a/include/samurai/timers.hpp +++ b/include/samurai/timers.hpp @@ -297,3 +297,4 @@ namespace samurai } } + From 262ba263cc97fb68cca94f4ad3f03b7a3d8697bd Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 15 Feb 2024 16:11:16 +0100 Subject: [PATCH 003/170] add timers on advection 2d demos --- demos/FiniteVolume/advection_2d.cpp | 42 ++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 4b6c85d0f..10d2d90f2 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -15,11 +15,13 @@ #include #include +#include + #include namespace fs = std::filesystem; template -auto init(Mesh& mesh) +auto init(Mesh& mesh, const double radius, const double x_center, const double y_center ) { auto u = samurai::make_field("u", mesh); @@ -28,9 +30,9 @@ auto init(Mesh& mesh) [&](auto& cell) { auto center = cell.center(); - const double radius = .2; - const double x_center = 0.3; - const double y_center = 0.3; + // const double radius = .2; + // const double x_center = 0.3; + // const double y_center = 0.3; if (((center[0] - x_center) * (center[0] - x_center) + (center[1] - y_center) * (center[1] - y_center)) <= radius * radius) { u[cell] = 1; @@ -146,18 +148,23 @@ void flux_correction(double dt, const std::array& a, const Field& u, template void save(const fs::path& path, const std::string& filename, const Field& u, const std::string& suffix = "") { - auto mesh = u.mesh(); - auto level_ = samurai::make_field("level", mesh); + auto mesh = u.mesh(); + auto level_ = samurai::make_field("level", mesh); + auto domain_ = samurai::make_field("domain", mesh); if (!fs::exists(path)) { fs::create_directory(path); } + boost::mpi::communicator world; + int mrank = world.rank(); + samurai::for_each_cell(mesh, [&](const auto& cell) { level_[cell] = cell.level; + domain_[ cell ] = mrank; }); #ifdef SAMURAI_WITH_MPI mpi::communicator world; @@ -171,12 +178,15 @@ int main(int argc, char* argv[]) { auto& app = samurai::initialize("Finite volume example for the advection equation in 2d using multiresolution", argc, argv); + Timers myTimers; + constexpr std::size_t dim = 2; using Config = samurai::MRConfig; // Simulation parameters + double radius = 0.2, x_center = 0.3, y_center = 0.3; xt::xtensor_fixed> min_corner = {0., 0.}; - xt::xtensor_fixed> max_corner = {1., 1.}; + xt::xtensor_fixed> max_corner = {4., 7.}; std::array a{ {1, 1} }; @@ -230,8 +240,11 @@ int main(int argc, char* argv[]) samurai::make_bc>(u, 0.); auto unp1 = samurai::make_field("unp1", mesh); + myTimers.start( "make_MRAdapt_init" ); auto MRadaptation = samurai::make_MRAdapt(u); MRadaptation(mr_epsilon, mr_regularity); + myTimers.stop( "make_MRAdapt_init" ); + save(path, filename, u, "_init"); std::size_t nsave = 1; @@ -239,7 +252,9 @@ int main(int argc, char* argv[]) while (t != Tf) { + myTimers.start( "MRadaptation" ); MRadaptation(mr_epsilon, mr_regularity); + myTimers.stop( "MRadaptation" ); t += dt; if (t > Tf) @@ -250,9 +265,16 @@ int main(int argc, char* argv[]) std::cout << fmt::format("iteration {}: t = {}, dt = {}", nt++, t, dt) << std::endl; + myTimers.start( "update_ghost_mr" ); samurai::update_ghost_mr(u); + myTimers.stop( "update_ghost_mr" ); + unp1.resize(); + + myTimers.start( "upwind" ); unp1 = u - dt * samurai::upwind(a, u); + myTimers.stop( "upwind" ); + if (correction) { flux_correction(dt, a, u, unp1); @@ -260,12 +282,18 @@ int main(int argc, char* argv[]) std::swap(u.array(), unp1.array()); + myTimers.start( "I/O" ); if (t >= static_cast(nsave + 1) * dt_save || t == Tf) { const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; save(path, filename, u, suffix); } + myTimers.stop( "I/O" ); + } + + myTimers.print(); + samurai::finalize(); return 0; } From 7027643a60a86f9555fc1401dddf7f49cdb0246b Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 15 Feb 2024 16:12:06 +0100 Subject: [PATCH 004/170] add load balancing demos (without MPI first) --- demos/CMakeLists.txt | 1 + demos/loadbalancing/CMakeLists.txt | 2 + demos/loadbalancing/partition.cpp | 64 ++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 demos/loadbalancing/CMakeLists.txt create mode 100644 demos/loadbalancing/partition.cpp diff --git a/demos/CMakeLists.txt b/demos/CMakeLists.txt index 886c62108..4dcafe466 100644 --- a/demos/CMakeLists.txt +++ b/demos/CMakeLists.txt @@ -3,6 +3,7 @@ find_package(CLI11) add_subdirectory(from_obj) add_subdirectory(FiniteVolume) +add_subdirectory(loadbalancing) # add_subdirectory(MPI) # add_subdirectory(LBM) add_subdirectory(p4est) diff --git a/demos/loadbalancing/CMakeLists.txt b/demos/loadbalancing/CMakeLists.txt new file mode 100644 index 000000000..8227a998a --- /dev/null +++ b/demos/loadbalancing/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(test-partition partition.cpp) +target_link_libraries(test-partition samurai CLI11::CLI11) diff --git a/demos/loadbalancing/partition.cpp b/demos/loadbalancing/partition.cpp new file mode 100644 index 000000000..0a9b8b873 --- /dev/null +++ b/demos/loadbalancing/partition.cpp @@ -0,0 +1,64 @@ +#include + +// CLI +#include + +// samurai +#include +#include +#include +#include +#include +#include +#include +#include + +int main( int argc, char * argv[] ){ + + samurai::initialize(argc, argv); + Timers myTimers; + + constexpr std::size_t dim = 2; + std::size_t minLevel = 3, maxLevel = 8; + double mr_regularity = 1.0, mr_epsilon = 1e-4; + xt::xtensor_fixed> minCorner = {0., 0.}; + xt::xtensor_fixed> maxCorner = {1., 1.}; + std::string filename = "load_balancing"; + + CLI::App app{"Load balancing test"}; + app.add_option("--min-corner", minCorner, "The min corner of the box")->capture_default_str()->group("Simulation parameters"); + app.add_option("--max-corner", maxCorner, "The max corner of the box")->capture_default_str()->group("Simulation parameters"); + app.add_option("--min-level", minLevel, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--max-level", maxLevel, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-eps", mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") + ->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-reg", mr_regularity,"The regularity criteria used by the multiresolution to " + "adapt the mesh")->capture_default_str()->group("Multiresolution"); + app.add_option("--filename", filename, "File name prefix")->capture_default_str()->group("Ouput"); + + CLI11_PARSE(app, argc, argv); + + std::size_t start_level = minLevel; + + myTimers.start("InitMesh"); + samurai::Box box( minCorner, maxCorner ); + samurai::CellArray mesh; + + mesh[start_level] = {start_level, box}; + + auto level = samurai::make_field( "level", mesh ); + auto rank = samurai::make_field( "rank", mesh ); + + samurai::for_each_cell( mesh, [&](const auto & cell ){ + level[cell] = static_cast( cell.level ); + rank [cell] = 0; + }); + myTimers.stop("InitMesh"); + + samurai::save( filename, mesh, level, rank ); + + myTimers.print(); + samurai::finalize(); + + return EXIT_SUCCESS; +} \ No newline at end of file From 8a560f87cd19d8e967dd85ab915f69fc6da1799e Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 15 Feb 2024 16:14:39 +0100 Subject: [PATCH 005/170] fix: wrong folder for sfc.hpp --- include/sfc.hpp | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 include/sfc.hpp diff --git a/include/sfc.hpp b/include/sfc.hpp deleted file mode 100644 index 0b3500a12..000000000 --- a/include/sfc.hpp +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include - -struct Logical_coord{ - uint32_t i, j, k; -}; - -template -class SFC { - - public: - using SFC_key_t = uint64_t; - - /** - * - * Interface to compute Space Filling Curve 1D index from - * 2D/3D logical coordinates of cell. - * - */ - template - inline SFC_key_t getKey( const Logical_coord & lc, const Args&... kw ) const{ - // return SFC_Flavor::getKey_impl( lc ); - if constexpr ( dim == 2 ) return static_cast(this)->getKey_2D_impl( lc, kw... ); - if constexpr ( dim == 3 ) return static_cast(this)->getKey_3D_impl( lc, kw... ); - }; - - /** - * - * Interface to compute logical coordinate from Space Filling Curve 1D index. - * - */ - template - inline Logical_coord getCoordinates( const SFC_key_t & clef ) const{ - // return SFC_Flavor::getKey_impl( lc ); - if constexpr ( dim == 2 ) return static_cast(this)->getCoordinates_2D_impl( clef ); - if constexpr ( dim == 3 ) return static_cast(this)->getCoordinates_3D_impl( clef ); - }; - -}; \ No newline at end of file From 70f017b7d15d61c25d32f6078154646c9d9bd751 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 15 Feb 2024 16:14:58 +0100 Subject: [PATCH 006/170] fix: correct folder for sfc.hpp --- include/samurai/sfc.hpp | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 include/samurai/sfc.hpp diff --git a/include/samurai/sfc.hpp b/include/samurai/sfc.hpp new file mode 100644 index 000000000..0b3500a12 --- /dev/null +++ b/include/samurai/sfc.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include + +struct Logical_coord{ + uint32_t i, j, k; +}; + +template +class SFC { + + public: + using SFC_key_t = uint64_t; + + /** + * + * Interface to compute Space Filling Curve 1D index from + * 2D/3D logical coordinates of cell. + * + */ + template + inline SFC_key_t getKey( const Logical_coord & lc, const Args&... kw ) const{ + // return SFC_Flavor::getKey_impl( lc ); + if constexpr ( dim == 2 ) return static_cast(this)->getKey_2D_impl( lc, kw... ); + if constexpr ( dim == 3 ) return static_cast(this)->getKey_3D_impl( lc, kw... ); + }; + + /** + * + * Interface to compute logical coordinate from Space Filling Curve 1D index. + * + */ + template + inline Logical_coord getCoordinates( const SFC_key_t & clef ) const{ + // return SFC_Flavor::getKey_impl( lc ); + if constexpr ( dim == 2 ) return static_cast(this)->getCoordinates_2D_impl( clef ); + if constexpr ( dim == 3 ) return static_cast(this)->getCoordinates_3D_impl( clef ); + }; + +}; \ No newline at end of file From abc7b56f4c3aa81b0989fcebf54c65910a61bf6c Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 15 Feb 2024 16:48:51 +0100 Subject: [PATCH 007/170] add some missing tests for samurai::cell --- tests/test_cell.cpp | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/test_cell.cpp b/tests/test_cell.cpp index 9893d47dd..49a00ef9c 100644 --- a/tests/test_cell.cpp +++ b/tests/test_cell.cpp @@ -50,4 +50,41 @@ namespace samurai xt::xarray expected{.5, .5}; EXPECT_EQ(c.corner(), expected); } -} + + /** + * Test of samurai::cell_length function (cell.hpp) + */ + TEST(cell, cell_length){ + EXPECT_DOUBLE_EQ(samurai::cell_length(1), 0.5); + EXPECT_DOUBLE_EQ(samurai::cell_length(3), 0.125); + EXPECT_DOUBLE_EQ(samurai::cell_length(20), 9.5367431640625e-07 ); + } + + /** + * Test of cell::face_center function (cell.hpp) + */ + TEST(cell, face_center){ + auto indices = xt::xtensor_fixed>({1, 1}); + samurai::Cell<2, Interval> c { 1, indices, 0 }; + + { + auto dir_x_p = xt::xtensor_fixed>({1, 0}); + auto fxp = c.face_center( dir_x_p ); + EXPECT_DOUBLE_EQ( fxp(0), 1. ); + } + + { + auto dir_x_m = xt::xtensor_fixed>({-1, 0}); + auto fxp = c.face_center( dir_x_m ); + EXPECT_DOUBLE_EQ( fxp(0), 0.5 ); + } + + { + auto dir_y_p = xt::xtensor_fixed>({0, 1}); + auto fxp = c.face_center( dir_y_p ); + EXPECT_DOUBLE_EQ( fxp(1), 1. ); + } + + } + +} \ No newline at end of file From 88d0e8a6f4f552bb4f1d2b1738f19cec93e6a43a Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Sun, 18 Feb 2024 16:07:48 +0100 Subject: [PATCH 008/170] add tests for MRmesh --- tests/test_mrmesh.cpp | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/test_mrmesh.cpp diff --git a/tests/test_mrmesh.cpp b/tests/test_mrmesh.cpp new file mode 100644 index 000000000..2d8a5a35a --- /dev/null +++ b/tests/test_mrmesh.cpp @@ -0,0 +1,60 @@ +#include +#include +#include + +#include +#include + +#include + +namespace samurai +{ + + /* + * test MR Mesh; + */ + TEST(mrmesh, test_nbcells){ + + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + + samurai::CellList cl; + cl[0][{0}].add_interval({0, 4}); + cl[0][{1}].add_interval({0, 1}); + cl[0][{1}].add_interval({3, 4}); + cl[0][{2}].add_interval({0, 1}); + cl[0][{2}].add_interval({3, 4}); + cl[0][{3}].add_interval({0, 3}); + + cl[1][{2}].add_interval({2, 6}); + cl[1][{3}].add_interval({2, 6}); + cl[1][{4}].add_interval({2, 4}); + cl[1][{4}].add_interval({5, 6}); + cl[1][{5}].add_interval({2, 6}); + cl[1][{6}].add_interval({6, 8}); + cl[1][{7}].add_interval({6, 7}); + + cl[2][{8}].add_interval({8, 10}); + cl[2][{8}].add_interval({8, 10}); + cl[2][{14}].add_interval({14, 16}); + cl[2][{15}].add_interval({14, 16}); + + Mesh_t mesh( cl, 2, 4 ); + + ASSERT_EQ( mesh.min_level(), 2 ); + ASSERT_EQ( mesh.max_level(), 4 ); + + std::vector nCellPerLevel = { 11, 18, 8 }; + for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ + std::cerr << "\t> NbCells level (" << ilvl << ") : " << mesh.nb_cells( 0 ) << std::endl; + ASSERT_EQ( mesh.nb_cells( ilvl - mesh.min_level() ), nCellPerLevel[ ilvl - mesh.min_level() ] ); + } + + std::cerr << "\t> Number of cells : " << mesh.nb_cells() << std::endl; + + + } + +} \ No newline at end of file From bd4c869877fb56d313fd3741a3211116e68bb0b9 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 6 Mar 2024 13:43:41 +0100 Subject: [PATCH 009/170] fix SFC computation for hilbert + interface --- include/samurai/hilbert.hpp | 263 ++++++++++++++++++++++++++++++------ include/samurai/morton.hpp | 38 +++--- include/samurai/sfc.hpp | 27 ++-- 3 files changed, 255 insertions(+), 73 deletions(-) diff --git a/include/samurai/hilbert.hpp b/include/samurai/hilbert.hpp index b6f5a6407..1ff9e1169 100644 --- a/include/samurai/hilbert.hpp +++ b/include/samurai/hilbert.hpp @@ -6,6 +6,8 @@ class Hilbert : public SFC { private: + // const int _nbits = sizeof(int) * 8; + const std::vector>> _2D_STATE = {{{1, 0, 2, 3}, {0, 0, 2, 2}}, {{0, 3, 2, 1}, {1, 3, 1, 3}}, {{2, 1, 0, 3}, {3, 1, 3, 1}}, @@ -46,63 +48,244 @@ class Hilbert : public SFC { {4,1,2,7,4,4,1,1,7,7,2,2}}}; + // def __init__(self, indexbits=16, dim=2): + // """ + // initialize hilbert class object + // """ + // assert indexbits > 0 + // assert 2 <= dim <= 3, "Dimension expected to be 2 or 3" + + // self._max_hindex = (2 ** indexbits) - 1 + // self._nbits = indexbits + // self._dimension = dim + + // # print("\t> Max H-index: 2^{}-1 = {} !".format(indexbits, self._max_hindex)) + + template + Coord_t AxestoTranspose( const Coord_t & c ) const { + + constexpr int _nbits = sizeof( uint32_t ) * 8; + + Coord_t _transposed; + for(size_t idim=0; idim 1 ){ + uint32_t p = q - 1; + for( size_t j = 0; j>= 1; + } + + // Gray code + for( int i=1; i 1 ){ + if( _transposed( dim - 1 ) & q ){ + t ^= ( q - 1 ); + } + q >>= 1; + } + + for(int i=0; i> 1 + // for i in range(self._dimension-1, 0, -1): + // x[i] ^= x[i-1] + // x[0] ^= t + + // q = 2 + // while q != m: + // p = q - 1 + // for i in range(self._dimension-1, -1, -1): + // if x[i] & q: + // x[0] ^= p + // else: + // t = (x[0] ^ x[i]) & p + // x[0] ^= t + // x[i] ^= t + // q <<= 1 + public: - /* - * Return the hilbert index from logical coordinate (i,j) or (i,j,k) - * - * Parameters: - * lc : structure containing logical coordinate - * level : used for multiresolution - * Return : - * key : hilbert key - */ - inline SFC_key_t getKey_2D_impl( const Logical_coord & lc, int lvl ) const { + template + inline SFC_key_t getKey_2D_impl( const Coord_t & lc ) const { + // SAMURAI_TRACE( "Hilbert::getKey_2D_impl Entering" ); - SFC_key_t la_clef = 0; - + // Be sure it is positive, shift must be done before + assert( lc( 0 ) >= 0 ); + assert( lc( 1 ) >= 0 ); + + constexpr SFC_key_t one = static_cast( 1 ); constexpr int dim = 2; - constexpr int twotondim = 1 << dim; + constexpr int _nbits = sizeof( lc( 0 ) ) * 8; - int ind_bits[lvl][dim]; - int h_digits[lvl]; + // std::cerr << "\t\t> Coord (" << coord(0) << ", " << coord(1) << std::endl; + xt::xtensor_fixed> coord = AxestoTranspose( lc ); + // std::cerr << "\t\t> Coord (" << coord(0) << ", " << coord(1) << std::endl; - SFC_key_t xyz[dim] = { lc.i, lc.j }; + int nb = ( _nbits * dim ) - 1; + SFC_key_t tmp = 0; - // Set ind_bits for current point - for (int ibit = 0; ibit < lvl; ++ibit) { - for( size_t idim=0; idim> ibit) & 1; + for(int jj=_nbits-1; jj>= 0; --jj){ + for( int ii=0; ii> jj; + // std::cerr << " jj : " << jj << ", a : " << a ; + + if( ( a & static_cast( 1 ) ) == 1 ){ + SFC_key_t ad = ( static_cast( 1 ) << nb ); + // std::cerr << " nb : " << nb; + tmp += ad; + } + nb --; } } - // Compute Hilbert key bits - int cur_state = 0; - int new_state; - for(int ibit=lvl-1; ibit>-1; --ibit) { + return tmp; + } - // Compute s_digit by interleaving bits - int s_digit = 0; - for(size_t idim=0; idim + inline SFC_key_t getKey_3D_impl( const Coord_t & lc ) const { + // SAMURAI_TRACE( "Hilbert::getKey_2D_impl Entering" ); - // Compute the new state from the state diagram - new_state = _2D_STATE[s_digit][0][cur_state]; - h_digits[ibit] = _2D_STATE[s_digit][1][cur_state]; + // Be sure it is positive, shift must be done before + assert( lc( 0 ) >= 0 ); + assert( lc( 1 ) >= 0 ); + assert( lc( 2 ) >= 0 ); - cur_state = new_state; - } + constexpr SFC_key_t one = static_cast( 1 ); + constexpr int dim = 3; + constexpr int _nbits = sizeof( lc( 0 ) ) * 8; - // Assemble the point's key - for(int ibit=0; ibit Coord (" << coord(0) << ", " << coord(1) << std::endl; + xt::xtensor_fixed> coord = AxestoTranspose( lc ); + // std::cerr << "\t\t> Coord (" << coord(0) << ", " << coord(1) << std::endl; + int nb = ( _nbits * dim ) - 1; + SFC_key_t tmp = 0; - return la_clef; + for(int jj=_nbits-1; jj>= 0; --jj){ + for( int ii=0; ii> jj; + // std::cerr << " jj : " << jj << ", a : " << a ; + + if( ( a & static_cast( 1 ) ) == 1 ){ + SFC_key_t ad = ( static_cast( 1 ) << nb ); + // std::cerr << " nb : " << nb; + tmp += ad; + } + nb --; + } + } + + return tmp; } - inline SFC_key_t getKey_3D_impl( const Logical_coord & lc, int lvl ) const { + /* + * Return the hilbert index from logical coordinate (i,j) or (i,j,k) + * + * Parameters: + * lc : structure containing logical coordinate + * level : used for multiresolution + * Return : + * key : hilbert key + */ + // inline SFC_key_t getKey_2D_impl_state( const Logical_coord & lc, int lvl ) const { + // SAMURAI_TRACE( "Hilbert::getKey_2D_impl Entering " ); + + // SFC_key_t la_clef = 0; + + // constexpr int dim = 2; + // constexpr int twotondim = 1 << dim; + + // std::vector> ind_bits( lvl, std::vector( dim ) ); + // std::vector h_digits( lvl, 0 ); + + // SFC_key_t xyz[dim] = { lc.i, lc.j }; + + // // Set ind_bits for current point + // for (int ibit = 0; ibit < lvl; ++ibit) { + // for( int idim=0; idim> ibit) & 1; + // } + // } + // SAMURAI_TRACE( "Hilbert::getKey_2D_impl set ind_bits " ); + + // // Compute Hilbert key bits + // int cur_state = 0; + // int new_state; + // for(int ibit=lvl-1; ibit>-1; --ibit) { + // SAMURAI_LOG( "ibit : " + std::to_string( ibit ) ); + + // // Compute s_digit by interleaving bits + // int s_digit = 0; + // for(int idim=0; idim + inline SFC_key_t getKey_3D_impl( const Coord_t & lc, int lvl ) const { SFC_key_t la_clef = 0; constexpr int dim = 3; @@ -111,7 +294,7 @@ class Hilbert : public SFC { int ind_bits[lvl][dim]; int h_digits[lvl]; - SFC_key_t xyz[dim] = { lc.i, lc.j, lc.k }; + SFC_key_t xyz[ dim ] = { lc( 0 ), lc( 1 ), lc( 2 ) }; // Set ind_bits for current point for (int ibit = 0; ibit < lvl; ++ibit) { diff --git a/include/samurai/morton.hpp b/include/samurai/morton.hpp index b937e8c7c..947dc5c3d 100644 --- a/include/samurai/morton.hpp +++ b/include/samurai/morton.hpp @@ -18,9 +18,9 @@ class Morton : public SFC { * return : * x : unsigned 64bits with splited by 3 */ - inline uint64_t splitBy3_2D( uint32_t logical_a ) const { + inline SFC_key_t splitBy3_2D( uint32_t logical_a ) const { - uint64_t x = logical_a & 0xffffffff; + SFC_key_t x = logical_a & 0xffffffff; x = (x | x << 16) & 0xffff0000ffff; x = (x | x << 8) & 0xff00ff00ff00ff; x = (x | x << 4) & 0xf0f0f0f0f0f0f0f; @@ -39,9 +39,9 @@ class Morton : public SFC { * return : * x : unsigned 64bits with splited by 3 */ - inline uint64_t splitBy3_3D(uint32_t logical_a) const { + inline SFC_key_t splitBy3_3D(uint32_t logical_a) const { - uint64_t x = logical_a & 0x1fffff; + SFC_key_t x = logical_a & 0x1fffff; x = (x | x << 32) & 0x1f00000000ffff; x = (x | x << 16) & 0x1f0000ff0000ff; @@ -65,15 +65,17 @@ class Morton : public SFC { * Return : * key : morton key */ - inline SFC_key_t getKey_2D_impl( const Logical_coord & lc ) const { + template + inline SFC_key_t getKey_2D_impl( const Coord_t & lc ) const { SFC_key_t la_clef = 0; - la_clef |= splitBy3_2D( lc.i ) | splitBy3_2D( lc.j ) << 1; + la_clef |= splitBy3_2D( lc( 0 ) ) | splitBy3_2D( lc( 1 ) ) << 1; return la_clef; } - inline SFC_key_t getKey_3D_impl( const Logical_coord & lc ) const { - uint64_t la_clef = 0; - la_clef |= splitBy3_3D( lc.i ) | splitBy3_3D( lc.j ) << 1 | splitBy3_3D( lc.k ) << 2; + template + inline SFC_key_t getKey_3D_impl( const Coord_t & lc ) const { + SFC_key_t la_clef = 0; + la_clef |= splitBy3_3D( lc( 0 ) ) | splitBy3_3D( lc( 1 ) ) << 1 | splitBy3_3D( lc( 2 ) ) << 2; return la_clef; } @@ -86,9 +88,9 @@ class Morton : public SFC { * Return : * lc (out): 2D/3D logical coordinates */ - inline Logical_coord getCoordinates_impl_2D( const SFC_key_t & clef ) const { + inline auto getCoordinates_2D_impl( const SFC_key_t & clef ) const { - Logical_coord lc {0, 0, 0}; + xt::xtensor_fixed> lc = { 0, 0 }; // Extract coord i SFC_key_t keyi = clef >> 0; @@ -98,7 +100,7 @@ class Morton : public SFC { keyi = (keyi ^ (keyi >> 4)) & 0x00ff00ff00ff00ff; keyi = (keyi ^ (keyi >> 8)) & 0x0000ffff0000ffff; keyi = (keyi ^ (keyi >> 16)) & 0x00000000ffffffff; - lc.i = static_cast(keyi); + lc( 0 ) = static_cast( keyi ); // extract coord j SFC_key_t keyj = clef >> 1; @@ -108,14 +110,14 @@ class Morton : public SFC { keyj = (keyj ^ (keyj >> 4)) & 0x00ff00ff00ff00ff; keyj = (keyj ^ (keyj >> 8)) & 0x0000ffff0000ffff; keyj = (keyj ^ (keyj >> 16)) & 0x00000000ffffffff; - lc.j = static_cast(keyj); + lc( 1 ) = static_cast( keyj ); return lc; } - inline Logical_coord getCoordinates_impl_3D( const SFC_key_t & clef ) const { + inline auto getCoordinates_3D_impl( const SFC_key_t & clef ) const { - Logical_coord lc = {0, 0, 0}; // k=0 + xt::xtensor_fixed> lc = { 0, 0, 0 }; // Extract coord i SFC_key_t keyi = clef >> 0; @@ -125,7 +127,7 @@ class Morton : public SFC { keyi = (keyi ^ (keyi >> 8)) & 0x00ff0000ff0000ff; keyi = (keyi ^ (keyi >> 16)) & 0x00ff00000000ffff; keyi = (keyi ^ (keyi >> 32)) & 0x1fffff; - lc.i = keyi; + lc( 0 ) = static_cast( keyi ); // assert for overflow ? SFC_key_t keyj = clef >> 1; keyj &= 0x1249249249249249; @@ -134,7 +136,7 @@ class Morton : public SFC { keyj = (keyj ^ (keyj >> 8)) & 0x00ff0000ff0000ff; keyj = (keyj ^ (keyj >> 16)) & 0x00ff00000000ffff; keyj = (keyj ^ (keyj >> 32)) & 0x1fffff; - lc.j = keyj; + lc( 1 ) = static_cast( keyj ); SFC_key_t keyk = clef >> 2; keyk &= 0x1249249249249249; @@ -143,7 +145,7 @@ class Morton : public SFC { keyk = (keyk ^ (keyk >> 8)) & 0x00ff0000ff0000ff; keyk = (keyk ^ (keyk >> 16)) & 0x00ff00000000ffff; keyk = (keyk ^ (keyk >> 32)) & 0x1fffff; - lc.k = keyk; + lc( 2 ) = static_cast( keyk ); return lc; } diff --git a/include/samurai/sfc.hpp b/include/samurai/sfc.hpp index 0b3500a12..ef67a4a45 100644 --- a/include/samurai/sfc.hpp +++ b/include/samurai/sfc.hpp @@ -2,15 +2,14 @@ #include -struct Logical_coord{ - uint32_t i, j, k; -}; +#include "assertLogTrace.hpp" + +using SFC_key_t = uint64_t; template class SFC { public: - using SFC_key_t = uint64_t; /** * @@ -18,12 +17,11 @@ class SFC { * 2D/3D logical coordinates of cell. * */ - template - inline SFC_key_t getKey( const Logical_coord & lc, const Args&... kw ) const{ - // return SFC_Flavor::getKey_impl( lc ); - if constexpr ( dim == 2 ) return static_cast(this)->getKey_2D_impl( lc, kw... ); - if constexpr ( dim == 3 ) return static_cast(this)->getKey_3D_impl( lc, kw... ); - }; + template + inline SFC_key_t getKey( const Coord_t & lc ) const{ + if constexpr ( dim == 2 ) return static_cast( this )->getKey_2D_impl( lc ); + if constexpr ( dim == 3 ) return static_cast( this )->getKey_3D_impl( lc ); + } /** * @@ -31,10 +29,9 @@ class SFC { * */ template - inline Logical_coord getCoordinates( const SFC_key_t & clef ) const{ - // return SFC_Flavor::getKey_impl( lc ); - if constexpr ( dim == 2 ) return static_cast(this)->getCoordinates_2D_impl( clef ); - if constexpr ( dim == 3 ) return static_cast(this)->getCoordinates_3D_impl( clef ); - }; + inline auto getCoordinates( const SFC_key_t & clef ) const{ + if constexpr ( dim == 2 ) return static_cast( this )->getCoordinates_2D_impl( clef ); + if constexpr ( dim == 3 ) return static_cast( this )->getCoordinates_3D_impl( clef ); + } }; \ No newline at end of file From 05dc1bb39554ae04eddfbd8ef8cc7a5a01e7a48b Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 6 Mar 2024 13:44:02 +0100 Subject: [PATCH 010/170] add unit test for sfc key computation --- tests/test_sfc.cpp | 188 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 tests/test_sfc.cpp diff --git a/tests/test_sfc.cpp b/tests/test_sfc.cpp new file mode 100644 index 000000000..497993924 --- /dev/null +++ b/tests/test_sfc.cpp @@ -0,0 +1,188 @@ +#include +#include +#include + +#include +#include + +#include + +namespace samurai +{ + using Coord_2D_t = xt::xtensor_fixed>; + using Coord_3D_t = xt::xtensor_fixed>; + + /* + * test computation of morton indexes 2D + */ + TEST(sfc, morton2D_getKey){ + constexpr int dim = 2; + + SFC morton; + + Coord_2D_t ij = { 0, 0 }; + auto key1 = morton.getKey( ij ); + ASSERT_EQ( key1, 0 ); + + ij = { 2, 1 }; + auto key2 = morton.getKey( ij ); + ASSERT_EQ( key2, 6 ); + + ij = { 5, 6 }; + auto key3 = morton.getKey( ij ); + ASSERT_EQ( key3, 57 ); + + ij = { 0, 4 }; + auto key4 = morton.getKey( ij ); + ASSERT_EQ( key4, 32 ); + + ij = { 4, 0 }; + auto key5 = morton.getKey( ij ); + ASSERT_EQ( key5, 16 ); + } + + /* + * test computation of (i,j) logical coordinate from morton indexes 2D + */ + TEST(sfc, morton2D_getCoordinates){ + constexpr int dim = 2; + + SFC morton; + + auto ij = morton.getCoordinates( static_cast( 0 ) ); + ASSERT_EQ( ij(0), 0 ); + ASSERT_EQ( ij(1), 0 ); + + auto ij2 = morton.getCoordinates( static_cast( 31 ) ); + ASSERT_EQ( ij2(0), 7 ); + ASSERT_EQ( ij2(1), 3 ); + + auto ij3 = morton.getCoordinates( static_cast( 51 ) ); + ASSERT_EQ( ij3(0), 5 ); + ASSERT_EQ( ij3(1), 5 ); + + auto ij4 = morton.getCoordinates( static_cast( 39 ) ); + ASSERT_EQ( ij4(0), 3 ); + ASSERT_EQ( ij4(1), 5 ); + } + + /* + * test computation of morton indexes 3D + */ + TEST(sfc, morton3D_getKey){ + constexpr int dim = 3; + + SFC morton; + + Coord_3D_t ijk = { 0, 0, 0 }; + auto key1 = morton.getKey( ijk ); + ASSERT_EQ( key1, 0 ); + + ijk = { 1, 1, 0 }; + auto key2 = morton.getKey( ijk ); + ASSERT_EQ( key2, 3 ); + + ijk = { 0, 0, 1 }; + auto key3 = morton.getKey( ijk ); + ASSERT_EQ( key3, 4 ); + + ijk = { 5, 9, 1 }; + auto key4 = morton.getKey( ijk ); + ASSERT_EQ( key4, 1095 ); + + } + + /* + * test computation of (i,j,k) logical coordinates from morton indexes 3D + */ + TEST(sfc, morton3D_getCoordinates){ + constexpr int dim = 3; + + SFC morton; + + auto ijk = morton.getCoordinates( static_cast( 1095 ) ); + ASSERT_EQ( ijk(0), 5 ); + ASSERT_EQ( ijk(1), 9 ); + ASSERT_EQ( ijk(2), 1 ); + + } + + /* + * test computation of hilbert key for 2D + */ + TEST(sfc, hilbert2D_getKey){ + constexpr int dim = 2; + + SFC hilbert; + + Coord_2D_t ij = { 0, 0 }; + ASSERT_EQ( hilbert.getKey( ij ), static_cast( 0 ) ); + + ij = { 1, 1 }; + ASSERT_EQ( hilbert.getKey( ij ), static_cast( 2 ) ); + + ij = { 1, 2 }; + ASSERT_EQ( hilbert.getKey( ij ), static_cast( 7 ) ); + + ij = { 3, 1 }; + ASSERT_EQ( hilbert.getKey( ij ), static_cast( 12 ) ); + + ij = { 3 , 4 }; + ASSERT_EQ( hilbert.getKey( ij ), static_cast( 31 ) ); + + ij = { 3 , 5 }; + ASSERT_EQ( hilbert.getKey( ij ), static_cast( 28 ) ); + + ij = { 3, 6 }; + ASSERT_EQ( hilbert.getKey( ij ), static_cast( 27 ) ); + + } + + /* + * test computation of hilbert key & back to {i,j,k} + */ + // TEST(tests_hilbert, test_consistency_hilbert3D){ + + // { + // Logical_Pos_t pt1 {0, 0 ,0}; + // auto hk1 = getHilbertKey3D( pt1, 0 ); + // auto pt1_r = getLogicalFromHilberKey3D( hk1, 0 ); + // ASSERT_EQ( pt1.i, pt1_r.i ); + // ASSERT_EQ( pt1.j, pt1_r.j ); + // ASSERT_EQ( pt1.k, pt1_r.k ); + // } + + // { + // Logical_Pos_t pt1 {2, 3, 1}; + // auto hk1 = getHilbertKey3D( pt1, 2 ); + // auto pt1_r = getLogicalFromHilberKey3D( hk1, 2 ); + // ASSERT_EQ( pt1.i, pt1_r.i ); + // ASSERT_EQ( pt1.j, pt1_r.j ); + // ASSERT_EQ( pt1.k, pt1_r.k ); + // } + + // { + // Logical_Pos_t pt1 {8, 14, 28}; + // auto hk1 = getHilbertKey3D( pt1, 6 ); + // auto pt1_r = getLogicalFromHilberKey3D( hk1, 6 ); + // ASSERT_EQ( pt1.i, pt1_r.i ); + // ASSERT_EQ( pt1.j, pt1_r.j ); + // ASSERT_EQ( pt1.k, pt1_r.k ); + // } + + // } + + /* + * test computation of morton indexes 3D + */ + // TEST(tests_hilbert, test_consistency_hilbert3D_old){ + + // { + // auto hk1 = get_hilbert_key_3D( {0, 0 ,0}, 0 ); + // auto pt1_r = get_logical_from_hilbert_3D( hk1, 0, 3); + // ASSERT_EQ( 0, pt1_r[0] ); + // ASSERT_EQ( 0, pt1_r[1] ); + // ASSERT_EQ( 0, pt1_r[2] ); + // } + +} \ No newline at end of file From 2c9f36d6e755583d2198917c44772972db462419 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Sat, 9 Mar 2024 09:48:41 +0100 Subject: [PATCH 011/170] add unit tests --- tests/test_mrmesh.cpp | 95 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 9 deletions(-) diff --git a/tests/test_mrmesh.cpp b/tests/test_mrmesh.cpp index 2d8a5a35a..f091910d3 100644 --- a/tests/test_mrmesh.cpp +++ b/tests/test_mrmesh.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -13,7 +14,7 @@ namespace samurai /* * test MR Mesh; */ - TEST(mrmesh, test_nbcells){ + TEST(mrmesh, test_nbcells2D){ constexpr int dim = 2; @@ -37,23 +38,99 @@ namespace samurai cl[1][{7}].add_interval({6, 7}); cl[2][{8}].add_interval({8, 10}); - cl[2][{8}].add_interval({8, 10}); + cl[2][{9}].add_interval({8, 10}); cl[2][{14}].add_interval({14, 16}); cl[2][{15}].add_interval({14, 16}); - Mesh_t mesh( cl, 2, 4 ); + Mesh_t mesh( cl, 0, 2 ); + + ASSERT_EQ( mesh.min_level(), 0 ); + ASSERT_EQ( mesh.max_level(), 2 ); - ASSERT_EQ( mesh.min_level(), 2 ); - ASSERT_EQ( mesh.max_level(), 4 ); + std::vector nCellPerLevel_withGhost = { 36, 48, 32 }; // including ghost + for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ + ASSERT_EQ( mesh.nb_cells( mesh.min_level() ), + nCellPerLevel_withGhost[ mesh.min_level() ] ); + } - std::vector nCellPerLevel = { 11, 18, 8 }; + std::vector nCellPerLevel_leaves = { 11, 18, 8 }; // not including ghost for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ - std::cerr << "\t> NbCells level (" << ilvl << ") : " << mesh.nb_cells( 0 ) << std::endl; - ASSERT_EQ( mesh.nb_cells( ilvl - mesh.min_level() ), nCellPerLevel[ ilvl - mesh.min_level() ] ); + ASSERT_EQ( mesh.nb_cells( mesh.min_level(), samurai::MRMeshId::cells ), + nCellPerLevel_leaves[ mesh.min_level() ] ); } - std::cerr << "\t> Number of cells : " << mesh.nb_cells() << std::endl; + } + + TEST(mrmesh, test_exist2D){ + + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + + samurai::CellList cl; + cl[0][{0}].add_interval({0, 4}); + cl[0][{1}].add_interval({0, 1}); + cl[0][{1}].add_interval({3, 4}); + cl[0][{2}].add_interval({0, 1}); + cl[0][{2}].add_interval({3, 4}); + cl[0][{3}].add_interval({0, 3}); + + cl[1][{2}].add_interval({2, 6}); + cl[1][{3}].add_interval({2, 6}); + cl[1][{4}].add_interval({2, 4}); + cl[1][{4}].add_interval({5, 6}); + cl[1][{5}].add_interval({2, 6}); + cl[1][{6}].add_interval({6, 8}); + cl[1][{7}].add_interval({6, 7}); + + cl[2][{8}].add_interval({8, 10}); + cl[2][{9}].add_interval({8, 10}); + cl[2][{14}].add_interval({14, 16}); + cl[2][{15}].add_interval({14, 16}); + + Mesh_t mesh( cl, 0, 2 ); + + Interval i{0, 3, 0}; + + // mesh.exists( samurai::MRMeshId::cells, 1, ) + } + + TEST(mrmesh, test_merge){ + + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using CellArray_t = Mesh_t::ca_type; + + samurai::CellList cl; + cl[0][{0}].add_interval({0, 4}); + cl[0][{1}].add_interval({0, 1}); + cl[0][{1}].add_interval({3, 4}); + + cl[1][{2}].add_interval({2, 6}); + + Mesh_t mesh( cl, 0, 2 ); + + std::cerr << "\t> Before merge : " << std::endl; + std::cerr << mesh << std::endl; + ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 10 ); + + samurai::CellList to_add_cl; + to_add_cl[1][{3}].add_interval({2, 6}); + + CellArray_t to_add_ca = { to_add_cl, false }; + + std::cerr << "\t> To merge : " << std::endl; + std::cerr << to_add_ca << std::endl; + + mesh.merge( to_add_ca ); + + std::cerr << "\t> After merge : " << std::endl; + std::cerr << mesh << std::endl; + ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 14 ); } From 0ca674466ef98a6e4227b0b36084f3fa7e4b6437 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 11 Mar 2024 10:30:39 +0100 Subject: [PATCH 012/170] add mesh remove test --- tests/test_mrmesh.cpp | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/tests/test_mrmesh.cpp b/tests/test_mrmesh.cpp index f091910d3..a5ec3f531 100644 --- a/tests/test_mrmesh.cpp +++ b/tests/test_mrmesh.cpp @@ -108,13 +108,9 @@ namespace samurai cl[0][{0}].add_interval({0, 4}); cl[0][{1}].add_interval({0, 1}); cl[0][{1}].add_interval({3, 4}); - cl[1][{2}].add_interval({2, 6}); Mesh_t mesh( cl, 0, 2 ); - - std::cerr << "\t> Before merge : " << std::endl; - std::cerr << mesh << std::endl; ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 10 ); samurai::CellList to_add_cl; @@ -122,16 +118,44 @@ namespace samurai CellArray_t to_add_ca = { to_add_cl, false }; - std::cerr << "\t> To merge : " << std::endl; - std::cerr << to_add_ca << std::endl; - mesh.merge( to_add_ca ); - std::cerr << "\t> After merge : " << std::endl; - std::cerr << mesh << std::endl; + // technically not sufficient to be sure the merge was done properly + ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 14 ); + + } + + TEST(mrmesh, test_remove){ + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using CellArray_t = Mesh_t::ca_type; + + samurai::CellList cl; + cl[0][{0}].add_interval({0, 4}); + cl[0][{1}].add_interval({0, 1}); + cl[0][{1}].add_interval({3, 4}); + cl[1][{2}].add_interval({2, 6}); + cl[1][{3}].add_interval({2, 6}); + + Mesh_t mesh( cl, 0, 2 ); ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 14 ); + samurai::CellList to_rm_cl; + to_rm_cl[0][{1}].add_interval({3,4}); + to_rm_cl[1][{3}].add_interval({2, 6}); + + CellArray_t to_rm_ca = { to_rm_cl, false }; + + mesh.remove( to_rm_ca ); + + // technically not sufficient to be sure the remove was done properly + ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 9 ); + } + + } \ No newline at end of file From 702fd44ba50502c550474c211c549418247a3de0 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:30:41 +0100 Subject: [PATCH 013/170] update statistisque module --- include/samurai/statistics.hpp | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/include/samurai/statistics.hpp b/include/samurai/statistics.hpp index 14a674ddb..5cd58327e 100644 --- a/include/samurai/statistics.hpp +++ b/include/samurai/statistics.hpp @@ -4,6 +4,7 @@ #include +#define WITH_STATS #if defined(WITH_STATS) #include using json = nlohmann::json; @@ -14,16 +15,20 @@ namespace samurai #if defined(WITH_STATS) struct Statistics { - Statistics(const std::string& filename, int save_all = 10) - : filename(filename) + Statistics( const std::string & filename, int save_all = 10) + : _outfile( filename ) , save_all(save_all) , icurrent(0) { + std::cerr << "\t> Constucteur [Statistics] " << std::endl; } template - void operator()(std::string test_case, const Mesh& mesh) + void operator()( std::string test_case, Mesh& mesh ) { + + std::cerr << "\t> Constucteur [operator()] " << std::endl; + icurrent++; using mesh_id_t = typename Mesh::mesh_id_t; auto ca = mesh[mesh_id_t::cells]; @@ -32,6 +37,10 @@ namespace samurai std::size_t min_level = ca.min_level(); std::size_t max_level = ca.max_level(); + size_t n_neighbours = mesh.mpi_neighbourhood().size(); + size_t n_cells_l = mesh.nb_cells( Mesh::mesh_id_t::cells ); + size_t n_cells_g = mesh.nb_cells( Mesh::mesh_id_t::reference ) - mesh.nb_cells( Mesh::mesh_id_t::cells ); + auto comp = [](const auto& a, const auto& b) { return a.size() < b.size(); @@ -72,6 +81,9 @@ namespace samurai stats[test_case].push_back({ {"min_level", min_level}, {"max_level", max_level}, + {"n_neighbours", n_neighbours}, + {"n_cells_g", n_cells_g}, // ghost ? + {"n_cells_l", n_cells_l}, // leaves {"by_level", by_level } }); } @@ -81,6 +93,9 @@ namespace samurai out.push_back({ {"min_level", min_level}, {"max_level", max_level}, + {"n_neighbours", n_neighbours}, + {"n_cells_g", n_cells_g}, // ghost ? + {"n_cells_l", n_cells_l}, // leaves {"by_level", by_level } }); stats[test_case] = out; @@ -88,7 +103,7 @@ namespace samurai if (icurrent == save_all) { - std::ofstream ofile(filename); + std::ofstream ofile( _outfile ); ofile << std::setw(4) << stats << std::endl; icurrent = 0; } @@ -96,17 +111,16 @@ namespace samurai ~Statistics() { - std::ofstream file(filename); + std::ofstream file( _outfile ); file << std::setw(4) << stats; } - std::string filename; + std::string _outfile; json stats; std::size_t icurrent; int save_all; }; - auto statistics = Statistics("stats.json"); #else template void statistics(const std::string& test_case, const Mesh& mesh){}; From e9f6ab5ca9f4d629dc7343be32c6025b48933ac8 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:32:04 +0100 Subject: [PATCH 014/170] add load balancing base class + some usefull functions --- include/samurai/load_balancing.hpp | 1296 ++++++++++++++++++++++++++++ 1 file changed, 1296 insertions(+) create mode 100644 include/samurai/load_balancing.hpp diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp new file mode 100644 index 000000000..660d275e7 --- /dev/null +++ b/include/samurai/load_balancing.hpp @@ -0,0 +1,1296 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// statistics +#include + +namespace samurai{ + + struct MPI_Load_Balance { + size_t _load; + std::vector neighbour; + std::vector load; + std::vector fluxes; + }; + + enum Distance_t { L1, L2, LINF, GRAVITY }; + enum Direction_t { FACE, DIAG, FACE_AND_DIAG }; + enum BalanceElement_t { CELL, INTERVAL }; + + /** + * Compute distance base on different norm. + */ + + template + static inline double distance_l2( const Coord_t & d1, const Coord_t & d2 ) { + double dist = 0.; + for( size_t idim=0; idim < static_cast( dim ); ++idim){ + double d = d1( idim ) - d2( idim ); + dist += d * d; + } + return std::sqrt( dist ); + } + + template + static inline double distance_inf( const Coord_t & d1, const Coord_t & d2 ) { + double dist = 0.; + for( size_t idim=0; idim < static_cast( dim ); ++idim){ + dist = std::max( std::abs( d1( idim ) - d2( idim ) ), dist ) ; + } + return dist; + } + + template + static inline double distance_l1( const Coord_t & d1, const Coord_t & d2 ) { + double dist = 0.; + for( size_t idim=0; idim < static_cast( dim ); ++idim ){ + dist += std::abs( d1( idim ) - d2( idim ) ) ; + } + return dist; + } + + /** + * Compute the load of the current process based on intervals or cells. It uses the + * mesh_id_t::cells to only consider leaves. + */ + template + static size_t cmptLoad( const Mesh_t & mesh ) { + + using mesh_id_t = typename Mesh_t::mesh_id_t; + + const auto & current_mesh = mesh[ mesh_id_t::cells ]; + + size_t current_process_load = 0; + + if constexpr ( elem == BalanceElement_t::CELL ) { + // cell-based load without weight. + samurai::for_each_interval( current_mesh, + [&]( std::size_t level, const auto& interval, const auto& index ){ + current_process_load += interval.size(); + }); + } else { + // interval-based load without weight + for( size_t level=current_mesh.min_level(); level<=current_mesh.max_level(); ++level ){ + current_process_load += current_mesh[ level ].shape()[ 0 ]; // only in x-axis ; + } + } + + return current_process_load; + } + + /** + * Compute fluxes based on load computing stategy based on graph with label + * propagation algorithm. Return, for the current process, the flux in term of + * load, i.e. the quantity of "load" to transfer to its neighbours. If the load + * is negative, it means that the process (current) must send load to neighbour, + * if positive it means that it must receive load. + * + * This function use 2 MPI all_gather calls. + * + */ + template + std::vector cmptFluxes( Mesh_t & mesh ) { + + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + + boost::mpi::communicator world; + + // give access to geometricaly neighbour process rank and mesh + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + size_t n_neighbours = neighbourhood.size(); + + // load of current process + int my_load = static_cast( cmptLoad( mesh ) ); + + // fluxes between processes + std::vector fluxes( n_neighbours, 0); + + // load of each process (all processes not only neighbours) + std::vector loads; + + // numbers of neighbours processes for each process, used for weighting fluxes + std::vector neighbourhood_n_neighbours; + + // get "my_load" from other processes + boost::mpi::all_gather(world, my_load, loads); + boost::mpi::all_gather(world, neighbourhood.size(), neighbourhood_n_neighbours); + + // compute updated my_load for current process based on its neighbourhood + int my_load_new = my_load; + for( std::size_t n_i = 0; n_i < n_neighbours; ++n_i ){ + + int neighbour_rank = neighbourhood[ n_i ].rank; + int neighbour_load = loads[ neighbour_rank ]; + double diff_load = static_cast( neighbour_load - my_load ); + + size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[ neighbour_rank ]; + + double weight = 1. / static_cast( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); + + int transfertLoad = static_cast( std::lround( weight * diff_load ) ); + + fluxes[ n_i ] += transfertLoad; + + my_load_new += transfertLoad; + } + + // Cannot have a negative or null load on an MPI process ! + assert( my_load_new > 0 ); + + return fluxes; + + } + + + /** Is gravity the key ? That would be awesome =) + * This is not a distance but well does not change anything to the algorithm + * - G m m' / R ? + * + * What should be m ? m' ? Let's G = 1. + * + **/ + template + static inline double gravity( const Coord_t & d1, const Coord_t & d2 ) { + double dist = distance_l2( d1, d2 ); + + // makes high level ( smallest ) cells to be exchanged more easily. 10 here is max + // level, hardcoded for tests. + // double m = 1./ static_cast( 1 << ( 10 - d1.level ) ) , m_p = 1.; + + // makes bigger cells to be exchanged more easily + // double m = 1./ static_cast( 1 << d1.level ) , m_p = 1.; + constexpr double G = 1.; + double m = 1., m_p = 1.; + double f = - G * m * m_p / ( dist * dist ); + + return f; + } + + // we are using Cell_t to allow ponderation using level; + template + inline constexpr double getDistance( const Coord_t & cc, const Coord_t & d ){ + static_assert( dim == 2 || dim == 3 ); + if constexpr ( dist == Distance_t::L1 ){ + return distance_l1( cc, d ); + }else if constexpr( dist == Distance_t::L2 ) { + return distance_l2( cc, d ); + }else if constexpr( dist == Distance_t::LINF ) { + return distance_inf( cc, d ); + }else if constexpr( dist == Distance_t::GRAVITY ){ + return gravity( cc, d ); + } + } + + template + class LoadBalancer { + + public: + + template + void load_balance( Mesh & mesh, Args&... kw ){ + static_cast(this)->load_balance_impl( mesh, kw... ); + } + + template + void evaluate_balancing( Mesh & mesh, Args&... kw ) const { + + boost::mpi::communicator world; + + if( world.rank() == 0 ) SAMURAI_TRACE( "[LoadBalancer::evaluate_balancing]::Entering function ... " ); + + std::vector load_cells, load_interval; + { + int my_load_i = static_cast( cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load_i, load_interval ); + + int my_load_c = static_cast( cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load_c, load_cells ); + } + + if( world.rank() == 0 ){ + std::cerr << "\t> LoadBalancer statistics : " << std::endl; + + std::vector::iterator min_load = std::min_element( load_cells.begin(), load_cells.end()); + auto rank = std::distance( load_cells.begin(), min_load ); + std::cerr << "\t\t> Min load {" << * min_load << ", cells } @ rank # " << rank << std::endl; + + std::vector::iterator max_load = std::max_element( load_cells.begin(), load_cells.end()); + rank = std::distance( load_cells.begin(), max_load ); + std::cerr << "\t\t> Max load {" << * max_load << ", cells } @ rank # " << rank << std::endl; + + + + } + + std::string _stats = fmt::format( "statistics_process_{}", world.rank() ); + samurai::Statistics s ( _stats ); + + s( "stats", mesh ); + + // s( "statistics" , mesh ); + + // no need to implement this in derived class, could be general. + // + // static_cast(this)->evaluate_impl( mesh, kw... ); + } + + }; + + /** + * Precomputed direction to face-to-face element for both 3D and 2D + */ + template + constexpr auto getDirectionFace() { + + using base_stencil = xt::xtensor_fixed>; + + xt::xtensor_fixed> stencils; + for( int ist=0; ist<2*dim; ++ist ){ + stencils[ ist ].fill( 0 ); + stencils[ ist ][ ist / 2 ] = 1 - ( ist % 2 ) * 2; + } + // if constexpr( dim == 2 ){ + // return xt::xtensor_fixed> {{{1, 0}, {-1, 0}, {0, 1}, {0, -1}}}; + // } + + return stencils; + } + + /** + * Precomputed direction to diagonal element for both 3D and 2D cases. + */ + template + constexpr auto getDirectionDiag() { + + using base_stencil = xt::xtensor_fixed>; + + static_assert( dim == 2 || dim == 3 ); + + if constexpr( dim == 2 ){ + return xt::xtensor_fixed> {{{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}}; + } + + if constexpr ( dim == 3 ){ + return xt::xtensor_fixed> { + {{ 1, 1, -1}, { 1, -1, -1}, + {-1, 1, -1}, {-1, -1, -1}, + { 1, 0, -1}, { -1, 0, -1}, + { 0, 1, -1}, { 0, -1, -1}, + { 1, 1, 0}, { 1, -1, 0}, + {-1, 1, 0}, {-1, -1, 0}, + { 1, 1, 1}, { 1, -1, 1}, + {-1, 1, 1}, {-1, -1, 1}, + { 1, 0, 1}, { -1, 0, 1}, + { 0, 1, 1}, { 0, -1, 1}}}; + } + + } + + /** + * Precompute direction to element in all direction face + diagonals: 26 in 3D, 8 in 2D + */ + template + constexpr auto getDirectionFaceAndDiag() { + + using base_stencil = xt::xtensor_fixed>; + + static_assert( dim == 2 || dim == 3 ); + + if constexpr( dim == 2 ){ + return xt::xtensor_fixed> {{{-1, 0}, { 1, 0}, + { 1, 1}, { 1, -1}, + { 0, -1}, { 0, 1}, + {-1, 1}, {-1, -1}}}; + } + + if constexpr ( dim == 3 ){ + return xt::xtensor_fixed> { + {{ 1, 0, 0}, {-1, 0, 0}, + { 0, 1, 0}, { 0, -1, 0}, + { 0, 0, 1}, { 0, 0, -1}, + { 1, 1, -1}, { 1, -1, -1}, + {-1, 1, -1}, {-1, -1, -1}, + { 1, 0, -1}, { -1, 0, -1}, + { 0, 1, -1}, { 0, -1, -1}, + { 1, 1, 0}, { 1, -1, 0}, + {-1, 1, 0}, {-1, -1, 0}, + { 1, 1, 1}, { 1, -1, 1}, + {-1, 1, 1}, {-1, -1, 1}, + { 1, 0, 1}, { -1, 0, 1}, + { 0, 1, 1}, { 0, -1, 1}}}; + } + + } + + template + inline auto getDirection(){ + static_assert( dim == 2 || dim == 3 ); + if constexpr ( dir == Direction_t::FACE ){ + return getDirectionFace(); + }else if constexpr( dir == Direction_t::DIAG ) { + return getDirectionDiag(); + }else if constexpr( dir == Direction_t::FACE_AND_DIAG ) { + return getDirectionFaceAndDiag(); + } + } + + /** + * Compute the barycenter of the mesh given in parameter. + * + * Feat: adjust wght which is hardcoded to 1. for now. + * by passing in parameters an array for example to modulate + * weight according to level + */ + template + xt::xtensor_fixed> _cmpCellBarycenter( Mesh_t & mesh ) { + + using Coord_t = xt::xtensor_fixed>; + + Coord_t bary; + bary.fill( 0. ); + + double wght_tot = 0.; + samurai::for_each_cell( mesh, [&]( auto & cell ){ + + // all cells equals + constexpr double wght = 1.; + + // higher weight for large cells + // double wght = 1. / static_cast( 1 << cell.level ); + + // higher weight for small cells + // double wght = 1. / static_cast( 1 << ( mesh.max_level() - cell.level ) ); + + auto cc = cell.center(); + + for(int idim=0; idim + static auto cmptInterface( Mesh_t & mesh, Mesh_t & omesh ){ + + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; + using mesh_id_t = typename Mesh_t::mesh_id_t; + + CellList_t interface; + + // operation are on leaves only + auto & currentMesh = mesh[ mesh_id_t::cells ]; + auto & otherMesh = omesh[ mesh_id_t::cells ]; + + // direction for translation to cmpt interface + auto dirs = getDirection(); + + for( size_t ist=0; ist( currentMesh.min_level() ), + static_cast( level ) - leveldiff ); + int maxlevel_check = std::min( static_cast( currentMesh.max_level() ), + static_cast( level ) + leveldiff ); + + for(size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + + // translate neighbour from dir (hopefully to current) all direction are tested + auto set = translate( otherMesh[ level ], stencil ); + auto intersect = intersection( set, currentMesh[ projlevel ] ).on( projlevel ); + + size_t nbInter_ = 0, nbCells = 0; + intersect([&]( auto & interval, auto & index ) { + nbInter_ += 1; + nbCells += interval.size(); + }); + + // we get more interval / cells, than wanted because neighbour has bigger cells + // 2:1 balance required here if not, need another loop... + if( nbInter_ > 0 && projlevel > level ){ + static_assert( leveldiff == 1 ); // for now at least + auto set_ = translate( currentMesh[ projlevel ], stencil ); + auto diff_ = difference( intersect, set_ ); + + diff_([&]( auto & interval, auto & index ) { + interface[ projlevel ][ index ].add_interval( interval ); + }); + + }else{ + if( nbInter_ > 0 ){ + intersect([&]( auto & interval, auto & index ) { + interface[ projlevel ][ index ].add_interval( interval ); + }); + } + } + + } + + } + + } + + CellArray_t interface_ = { interface, false }; + + return interface_; + + } + + /** + * Compute cells at the interface between geometricaly adjacent + * domains. It relies on neighbourhood mpi_subdomain_t data structure + * computed in Mesh_t. + * + * It returns a vector containing the number of cells (nc) and number of + * intervals (ni) for each neighbour (0 to n), in that order. + * + * Example: nc_0, ni_0, nc_1, ni_1, ... + * + */ + // template + // void _computeCartesianInterface( Mesh_t & mesh, Field_t & field ){ + template + static auto _computeCartesianInterface( Mesh_t & mesh ){ + + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; + using mesh_id_t = typename Mesh_t::mesh_id_t; + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + + // give access to geometricaly neighbour process rank and mesh + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + int n_neighbours = static_cast( neighbourhood.size() ); + + std::vector interface( n_neighbours ); + + // operation are on leaves only + auto currentMesh = mesh[ mesh_id_t::cells ]; + + // looks like using only FACE is better than using DIAG or FACE_AND_DIAG + auto dirs = getDirection(); + + for( int nbi=0; nbi( currentMesh.min_level() ), + static_cast( level ) - 1 ); + int maxlevel_check = std::min( static_cast( currentMesh.max_level() ), + static_cast( level ) + 1 ); + + for(size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + + // translate neighbour from dir (hopefully to current) all direction are tested + auto set = translate( neighbour_mesh[ level ], stencil ); + auto intersect = intersection( set, currentMesh[ projlevel ] ).on( projlevel ); + + size_t nbInter_ = 0, nbCells_ = 0; + intersect([&]( auto & interval, auto & index ) { + nbInter_ += 1; + nbCells_ += interval.size(); + }); + + // we get more interval / cells, than wanted because neighbour has bigger cells + if( nbInter_ > 0 && projlevel > level ){ + auto set_ = translate( currentMesh[ projlevel ], stencil ); + auto diff_ = difference( intersect, set_ ); + + nbInter_ = 0; + nbCells_ = 0; + diff_([&]( auto & interval, auto & index ) { + nbInter_ += 1; + nbCells_ += interval.size(); + // field( projlevel, i, index ) = world.rank() * 10; + interface[ nbi ][ projlevel ][ index ].add_interval( interval ); + }); + + }else{ + if( nbInter_ > 0 ){ + intersect([&]( auto & interval, auto & index ) { + interface[ nbi ][ projlevel ][ index ].add_interval( interval ); + }); + } + } + + nbCellsIntersection = nbCells_ ; + nbIntervalIntersection = nbInter_ ; + + } + + } + + } + + } + + std::vector interface_( n_neighbours ); + for(size_t nbi=0; nbi & all ){ + + for( size_t irank = 0; irank < all.size(); ++irank ){ + + // number of cells + // supposing each cell has a cost of 1. ( no level dependency ) + size_t load = all[ irank ]._load; + + int n_neighbours = all[ irank ].neighbour.size(); + + { + std::cerr << "[compute_load_balancing_fluxes] Process # " << irank << " load : " << load << std::endl; + std::cerr << "[compute_load_balancing_fluxes] Process # " << irank << " nneighbours : " << n_neighbours << std::endl; + std::cerr << "[compute_load_balancing_fluxes] Process # " << irank << " neighbours : "; + for( size_t in=0; in loads; + + // data "load" to transfer to neighbour processes + all[ irank ].fluxes.resize( n_neighbours ); + std::fill( all[ irank ].fluxes.begin(), all[ irank ].fluxes.end(), 0); + + const std::size_t n_iterations = 1; + + for (std::size_t k = 0; k < n_iterations; ++k) { + + // numbers of neighboors processes for each neighbour process + std::vector nb_neighbours; + + if( irank == 0 ) std::cerr << "[compute_load_balancing_fluxes] Fluxes iteration # " << k << std::endl; + + // // get info from processes + // mpi::all_gather(world, load, loads); + // mpi::all_gather(world, m_mpi_neighbourhood.size(), nb_neighbours); + + // load of current process + int64_t load_np1 = load; + + // compute updated load for current process based on its neighbourhood + for (std::size_t j_rank = 0; j_rank < n_neighbours; ++j_rank){ + + auto neighbour_rank = all[ irank ].neighbour[ j_rank ]; + auto neighbour_load = all[ irank ].load[ j_rank ]; + int64_t diff_load = neighbour_load - load; + + int nb_neighbours_neighbour = all[ neighbour_rank ].neighbour.size(); + + double weight = 1. / ( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); + + int64_t transfertLoad = static_cast( std::lround( weight * diff_load ) ); + + all[ irank].fluxes[ j_rank ] += transfertLoad; + + load_np1 += transfertLoad; + } + + // do check on load & fluxes ? + + { + std::cerr << "fluxes : "; + for( size_t in=0; in(load_np1); + } + + } + + } + + /** + * Check if there is an intersection between two meshes. We move one mesh into the direction + * of the other based on "dir" stencils (template parameter). Here, we rely on the 2:1 balance. + * By default, stencils are "1" based, which means that we move the mesh by one unit element. This + * could be changed if necessary by multiplying the stencils by an integer value. + */ + template + bool intersectionExists( Mesh_t & meshA, Mesh_t & meshB ){ + + using mesh_id_t = typename Mesh_t::mesh_id_t; + + // operation are on leaves cells only + auto meshA_ = meshA[ mesh_id_t::cells ]; + auto meshB_ = meshB[ mesh_id_t::cells ]; + + // get stencils for direction + auto dirs = getDirection(); + + for( size_t ist=0; ist( meshA_.min_level() ), + static_cast( level ) - 1 ); + int maxlevel_check = std::min( static_cast( meshA_.max_level() ), + static_cast( level ) + 1 ); + + for(int projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + + // translate meshB_ in "dir" direction + auto set = translate( meshB_[ level ], stencil ); + auto intersect = intersection( set, meshA_[ projlevel ] ).on( projlevel ); + + size_t nbInter_ = 0; + intersect([&]( auto & interval, auto & index ) { + nbInter_ += 1; + }); + + if( nbInter_ > 0 ) return true; + + nbIntervalIntersection += nbInter_ ; + + } + + } + + } + + return false; + + } + + /** + * Discover new neighbour connection that might arise during load balancing + */ + template + bool discover_neighbour( Mesh_t & mesh ) { + + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + + boost::mpi::communicator world; + + // DEBUG + std::ofstream logs; + logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs << "# discover_neighbour" << std::endl; + + // give access to geometricaly neighbour process rank and mesh + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + int n_neighbours = static_cast( neighbourhood.size() ); + + bool requireGeneralUpdate = false; + + std::vector keepNeighbour( neighbourhood.size(), true ); + + std::vector> neighbourConnection( neighbourhood.size() ); + + // for each neighbour we check if there is an intersection with another one. + for( size_t nbi=0; nbi( mesh, neighbourhood[ nbi ].mesh ); + + // require update if a connection is lost + requireGeneralUpdate = requireGeneralUpdate || ( ! keepNeighbour[ nbi ] ); + + if( ! keepNeighbour[ nbi ] ){ + logs << fmt::format("Loosing neighbour connection with {}", neighbourhood[ nbi ].rank) << std::endl; + } + + // check neighbour - neighbour connection + for( size_t nbj=nbi+1; nbj {}", neighbourhood[ nbi ].rank, + neighbourhood[ nbj ].rank ) << std::endl; + bool connected = intersectionExists( neighbourhood[ nbi ].mesh, neighbourhood[ nbj ].mesh ); + + if( connected ) { + neighbourConnection[ nbi ].emplace_back( neighbourhood[ nbj ].rank ); + neighbourConnection[ nbj ].emplace_back( neighbourhood[ nbi ].rank ); + } + } + } + + // communicate to neighbours the list of connection they have + for( size_t nbi=0; nbi Sending computed neighbour: {"; + for( const auto & i : neighbourConnection[ nbi ] ) + logs << i << ", "; + logs << "} to process " << neighbourhood[ nbi ].rank << std::endl; + // end debug + + world.send( neighbourhood[ nbi ].rank, 28, neighbourConnection[ nbi ] ); + } + + // map of current MPI neighbours processes rank + std::map _tmp; // key = rank, value = required + for( size_t nbi=0; nbi Receiving computed neighbour connection list from neighbour processes " << std::endl; + for( size_t nbi=0; nbi _rcv_neighbour_list; + world.recv( neighbourhood[ nbi ].rank, 28, _rcv_neighbour_list ); + + for(size_t in=0; in<_rcv_neighbour_list.size(); ++in ){ + if( _tmp.find( _rcv_neighbour_list[ in ] ) == _tmp.end() ){ + logs << "\t\t> New neighbour detected : " << _rcv_neighbour_list[ in ] << std::endl; + + _tmp[ _rcv_neighbour_list[ in ] ] = true; + requireGeneralUpdate = requireGeneralUpdate || true; + } + } + + } + + world.barrier(); + + // update neighbours ranks + neighbourhood.clear(); + for( const auto & ni : _tmp ){ + neighbourhood.emplace_back( ni.first ); + } + + // debug + logs << "New neighbourhood : {"; + for( size_t nbi=0; nbi +void perform_load_balancing_SFC( Mesh & mesh, int ndomains, Field_t & fake_mpi_rank ) { + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using cell_t = typename Mesh_t::cell_t; + + SFC sfc; + + // SFC key (used for implicit sorting through map mechanism) + std::map sfc_map; + + int sfc_max_level = mesh.max_level(); // for now but remember must <= 21 for Morton + + SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); + + samurai::for_each_cell( mesh, [&]( const auto & cell ){ + + // ij coordinate of cell + double dxmax = samurai::cell_length( sfc_max_level ); + + auto tmp = cell.center() / dxmax; + + xt::xtensor_fixed> ij = { static_cast( tmp( 0 ) ), + static_cast( tmp( 1 ) ) }; + + auto key = sfc.template getKey( ij ); + // std::cerr << "\t> Coord (" << ij( 0 ) << ", " << ij( 1 ) << ") ----> " << key << std::endl; + + sfc_map[ key ] = cell ; + + }); + + size_t nbcells_tot = mesh.nb_cells( Mesh::mesh_id_t::cells ); //load -balancing on leaves + size_t ncellsPerProc = std::floor( nbcells_tot / ndomains ); + + // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; + std::cerr << "\t> Total number of cells : " << nbcells_tot << std::endl; + std::cout << "\t> Perfect load-balancing (weight-cell = 1.) : " << static_cast( nbcells_tot / ndomains ) + << " / MPI" << std::endl; + + + size_t cindex = 0; + for(const auto & item : sfc_map ) { + auto sfc_key = item.first; + + int fake_rank = std::floor( cindex / ncellsPerProc ); + if ( fake_rank >= ndomains ) fake_rank = ndomains - 1; + + fake_mpi_rank[ item.second ] = fake_rank; + cindex ++; + } + +} + +enum Interval_CellPosition { FIRST, LAST }; + +template +void perform_load_balancing_SFC_Interval( Mesh & mesh, int ndomains, Field_t & fake_mpi_rank, const Interval_CellPosition &icp ) { + + using inter_t = samurai::Interval; + + struct Data { + size_t level; + inter_t interval; + xt::xtensor_fixed> indices; + }; + + // SFC key (used for implicit sorting through map mechanism) + std::map sfc_map; + + SFC sfc; + int sfc_max_level = mesh.max_level(); // for now but remember must <= 21 for Morton + + SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); + + size_t ninterval = 0; + samurai::for_each_interval( mesh, [&]( std::size_t level, const auto& inter, const auto& index ){ + + // get Logical coordinate or first cell + xt::xtensor_fixed> icell; + + icp == Interval_CellPosition::LAST ? icell( 0 ) = inter.end - 1 : icell( 0 ) = inter.start; + + for(int idim=0; idim> ijk; + + if constexpr ( dim == 2 ) ijk = { static_cast( icell( 0 ) ), + static_cast( icell( 1 ) ) }; + + if constexpr ( dim == 3 ) ijk = { static_cast( icell( 0 ) ), + static_cast( icell( 1 ) ), + static_cast( icell( 2 ) ) }; + + sfc_map[ sfc.getKey( ijk ) ] = { level, inter, index }; + + ninterval ++; + }); + + size_t ninterPerProc = std::floor( ninterval / ndomains ); + + // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; + std::cerr << "\t> Total number of interval : " << ninterval << std::endl; + std::cout << "\t> Perfect load-balancing (weight-interval = 1.) : " << static_cast( ninterPerProc ) + << " / MPI" << std::endl; + + + size_t cindex = 0; + for(const auto & item : sfc_map ) { + int fake_rank = std::floor( cindex / ninterPerProc ); + if ( fake_rank >= ndomains ) fake_rank = ndomains - 1; + + fake_mpi_rank( item.second.level, item.second.interval, item.second.indices ) = fake_rank; + + cindex ++; + } + +} + +/** + * + * Global contains the global mesh. Since this is a toy function, it contains the union of all mesh. + * meshes contains the mesh of each MPI process ( or let say the mesh of neighbour processes ...) + * +*/ +template +void perform_load_balancing_diffusion( AMesh_t & global, std::vector & meshes, int ndomains, + const std::vector & all, Field_t & fake_mpi_rank ) { + + using CellList_t = typename AMesh_t::cl_type; + + struct Coord_t { xt::xtensor_fixed> coord; }; + + std::vector barycenters ( ndomains ); + + { // compute barycenter of current domains ( here, all domains since with simulate multiple MPI domains) + + int maxlevel = 4; + for( size_t m_=0; m_ < meshes.size(); ++m_ ) { + + double wght_tot = 0.; + samurai::for_each_cell( meshes[ m_ ], [&]( const auto & cell ) { + + // [OPTIMIZATION] precompute weight as array + double wght = 1. / ( 1 << ( maxlevel - cell.level ) ); + + const auto cc = cell.center(); + + barycenters[ m_ ].coord( 0 ) += cc( 0 ) * wght; + barycenters[ m_ ].coord( 1 ) += cc( 1 ) * wght; + if constexpr ( dim == 3 ) { barycenters[ m_ ].coord( 2 ) += cc( 2 ) * wght; } + + wght_tot += wght; + + }); + + barycenters[ m_ ].coord( 0 ) /= wght_tot; + barycenters[ m_ ].coord( 1 ) /= wght_tot; + if constexpr ( dim == 3 ) barycenters[ m_ ].coord( 2 ) /= wght_tot; + + std::cerr << "\t> Domain # " << m_ << ", bc : {" << barycenters[ m_ ].coord( 0 ) << ", " + << barycenters[ m_ ].coord( 1 ) << "}" << std::endl; + + } + + } + + std::vector new_meshes( ndomains ); + std::vector exchanged( ndomains ); + + for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains + + std::cerr << "\t> Working on domains # " << m_ << std::endl; + + // auto dist = samurai::make_field( "rank", meshes[ m_ ] ); + + // id des neighbours dans le tableau de la structure MPI_Load_Balance + // attention différent du rank mpi ! + std::vector id_send; + + int n_neighbours = static_cast( all[ m_ ].neighbour.size() ); + for(std::size_t nbi = 0; nbi Neighbour rank : " << nbi_rank << std::endl; + std::cerr << "\t\t> Neighbour flux : " << all[ m_ ].fluxes[ nbi ] << std::endl; + + if( nbCellsToTransfer < 0 ){ + + id_send.emplace_back( nbi ); + + // Logical_coord_t stencil; + // { // Compute the stencil or normalized direction to neighbour + // Coord_t tmp; + // double n2 = 0; + // for(int idim = 0; idim( tmp.coord( idim ) / 0.5 ); + // } + + // std::cerr << "\t\t> stencil for neighbour # " << nbi_rank << " : "; + // for(size_t idim=0; idim Number RECV : " << id_send.size() << std::endl; + + std::vector already_given( n_neighbours, 0 ); + + for_each_interval( meshes[ m_ ], [&]( std::size_t level, const auto& interval, const auto& index ){ + Coord_t ibar; + + std::cerr << "\t\t> Interval [" << interval.start << ", " << interval.end << "[" << std::endl; + + double dm = 1. / (1 << level ); + ibar.coord( 0 ) = ( (interval.end - interval.start) * 0.5 + interval.start ) * dm ; + ibar.coord( 1 ) = ( index( 1 ) * dm ) + dm * 0.5; + if constexpr ( dim == 3 ) ibar( 2 ) = ( index( 2 ) * dm ) + dm * 0.5; + + std::cerr << "\t\t\t> level " << level << " center @ {" << ibar.coord(0) << ", " << ibar.coord(1) << "}" << std::endl; + + // find which neighbour will potentially receive this interval + int winner_id = -1; // keep it to current if still negative + double winner_dist = std::numeric_limits::max(); + for( int nbi=0; nbi Dist to neighbour #" << neighbour_rank << " : " << dist << " vs " << winner_dist << std::endl; + std::cerr << "\t\t\t> Already given to this neighbour " << already_given[ neighbour_rank ] << std::endl; + std::cerr << "\t\t\t> NbCells of this interval " << interval.size() << std::endl; + std::cerr << "\t\t\t> fluxes for this neighbour : " << all[ m_ ].fluxes[ id_send[ nbi ] ] << std::endl; + + if( dist < winner_dist && + already_given[ neighbour_rank ] + interval.size() <= (- all[ m_ ].fluxes[ id_send[ nbi ] ]) ){ + + winner_id = id_send[ nbi ]; + winner_dist = dist; + } + + } + + if( winner_id >= 0 ){ + auto neighbour_rank = all[ m_ ].neighbour[ winner_id ]; + std::cerr << "\t> Interval given to process " << neighbour_rank << " + ncells : " << interval.size() << std::endl; + exchanged[ neighbour_rank ][ level ][ index ].add_interval( interval ); + already_given[ neighbour_rank ] += interval.size(); + }else{ + new_meshes[ m_ ][ level ][ index ].add_interval( interval ); + } + + }); + + + meshes[ m_ ] = { new_meshes[ m_ ], true }; + + } + + for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains + for( int level=global.min_level(); level<=global.max_level(); ++level ) { + auto intersect = intersection( global[ level ], meshes[ m_ ][ level ]); + + intersect([&]( auto & i, auto & index ) { + fake_mpi_rank( level, i, index ) = static_cast( m_ ); + }); + } + + AMesh_t tmp = { exchanged[ m_], true }; + for( int level=global.min_level(); level<=global.max_level(); ++level ) { + auto intersect = intersection( global[ level ], tmp[ level ]); + + intersect([&]( auto & i, auto & index ) { + fake_mpi_rank( level, i, index ) = static_cast( m_ ); + }); + } + } + +} + +// template +// void perform_load_balancing_diffusion( AMesh_t & global, int ndomains, const std::vector & all, Field_t & fake_mpi_rank ) { + +// using CellList_t = typename AMesh_t::cl_type; +// using cell_t = typename AMesh_t::cell_t; + +// struct Coord_t { xt::xtensor_fixed> coord; }; +// struct Logical_coord_t { xt::xtensor_fixed> coord; }; + +// std::vector barycenters ( ndomains ); + +// { // compute barycenter of current domains ( here, all domains since with simulate multiple MPI domains) + +// int maxlevel = 4; +// for( size_t m_=0; m_ < meshes.size(); ++m_ ) { + +// double wght_tot = 0.; +// samurai::for_each_cell( meshes[ m_ ], [&]( const auto & cell ) { + +// // [OPTIMIZATION] precompute weight as array +// double wght = 1. / ( 1 << ( maxlevel - cell.level ) ); + +// const auto cc = cell.center(); + +// barycenters[ m_ ].coord( 0 ) += cc( 0 ) * wght; +// barycenters[ m_ ].coord( 1 ) += cc( 1 ) * wght; +// if constexpr ( dim == 3 ) { barycenters[ m_ ].coord( 2 ) += cc( 2 ) * wght; } + +// wght_tot += wght; + +// }); + +// barycenters[ m_ ].coord( 0 ) /= wght_tot; +// barycenters[ m_ ].coord( 1 ) /= wght_tot; +// if constexpr ( dim == 3 ) barycenters[ m_ ].coord( 2 ) /= wght_tot; + +// std::cerr << "\t> Domain # " << m_ << ", bc : {" << barycenters[ m_ ].coord( 0 ) << ", " +// << barycenters[ m_ ].coord( 1 ) << "}" << std::endl; + +// } + +// } + +// std::vector new_meshes( ndomains ); +// std::vector exchanged( ndomains ); + +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains + +// std::cerr << "\t> Working on domains # " << m_ << std::endl; + +// auto dist = samurai::make_field( "rank", meshes[ m_ ] ); + +// // neighbours offset to which data must be sent. +// // This is not equivalent to the MPI rank +// std::vector id_send; + +// int n_neighbours = static_cast( all[ m_ ].neighbour.size() ); +// for(std::size_t nbi = 0; nbi already_given( n_neighbours, 0 ); + +// // loop over each interval of the current ( or meshes[ m_ ] ) domain and check if it needs to be sent +// // to a given neighbour +// for_each_interval( meshes[ m_ ], [&]( std::size_t level, const auto& interval, const auto& index ){ +// Coord_t ibar; + +// std::cerr << "\t\t> Interval [" << interval.start << ", " << interval.end << "[" << std::endl; + +// double dm = 1. / (1 << level ); +// ibar.coord( 0 ) = ( (interval.end - interval.start) * 0.5 + interval.start ) * dm ; +// ibar.coord( 1 ) = ( index( 1 ) * dm ) + dm * 0.5; +// if constexpr ( dim == 3 ) ibar( 2 ) = ( index( 2 ) * dm ) + dm * 0.5; + +// std::cerr << "\t\t\t> level " << level << " center @ {" << ibar.coord(0) << ", " << ibar.coord(1) << "}" << std::endl; + +// // find which neighbour will potentially receive this interval +// int winner_id = -1; // keep it to current if still negative +// double winner_dist = std::numeric_limits::max(); +// for( int nbi=0; nbi Dist to neighbour #" << neighbour_rank << " : " << dist << " vs " << winner_dist << std::endl; +// std::cerr << "\t\t\t> Already given to this neighbour " << already_given[ neighbour_rank ] << std::endl; +// std::cerr << "\t\t\t> NbCells of this interval " << interval.size() << std::endl; +// std::cerr << "\t\t\t> fluxes for this neighbour : " << all[ m_ ].fluxes[ id_send[ nbi ] ] << std::endl; + +// if( dist < winner_dist && +// already_given[ neighbour_rank ] + interval.size() <= (- all[ m_ ].fluxes[ id_send[ nbi ] ]) ){ + +// winner_id = id_send[ nbi ]; +// winner_dist = dist; +// } + +// } + +// if( winner_id >= 0 ){ +// auto neighbour_rank = all[ m_ ].neighbour[ winner_id ]; +// std::cerr << "\t> Interval given to process " << neighbour_rank << " + ncells : " << interval.size() << std::endl; +// exchanged[ neighbour_rank ][ level ][ index ].add_interval( interval ); +// already_given[ neighbour_rank ] += interval.size(); +// }else{ +// new_meshes[ m_ ][ level ][ index ].add_interval( interval ); +// } + +// }); + + +// meshes[ m_ ] = { new_meshes[ m_ ], true }; + +// } + +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains +// for( int level=global.min_level(); level<=global.max_level(); ++level ) { +// auto intersect = intersection( global[ level ], meshes[ m_ ][ level ]); + +// intersect([&]( auto & i, auto & index ) { +// fake_mpi_rank( level, i, index ) = static_cast( m_ ); +// }); +// } + +// AMesh_t tmp = { exchanged[ m_], true }; +// for( int level=global.min_level(); level<=global.max_level(); ++level ) { +// auto intersect = intersection( global[ level ], tmp[ level ]); + +// intersect([&]( auto & i, auto & index ) { +// fake_mpi_rank( level, i, index ) = static_cast( m_ ); +// }); +// } +// } + +// } \ No newline at end of file From 2a012ad69110ab3d4e55be981a8df1c130169cb5 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:32:46 +0100 Subject: [PATCH 015/170] add load balancing derived class for SFC --- include/samurai/load_balancing_sfc.hpp | 214 +++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 include/samurai/load_balancing_sfc.hpp diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp new file mode 100644 index 000000000..b6b8b5bb4 --- /dev/null +++ b/include/samurai/load_balancing_sfc.hpp @@ -0,0 +1,214 @@ +#pragma once + +#include "assertLogTrace.hpp" + +#include "load_balancing.hpp" + +template +class SFC_LoadBalancer_cells : public samurai::LoadBalancer> { + + private: + SFC_type_t _sfc; + + public: + template + void load_balance_impl( Mesh & mesh ){ + _sfc.getIndex( 1, 2 ); + } + +}; + +template +class SFC_LoadBalancer_interval : public samurai::LoadBalancer> { + + private: + SFC_type_t _sfc; + int _ndomains; + int _rank; + + public: + + SFC_LoadBalancer_interval() { + +#ifdef SAMURAI_WITH_MPI + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); +#else + _ndomains = 1; + _rank = 0; +#endif + }; + + template + void load_balance_impl( Mesh & mesh ){ + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using cell_t = typename Mesh_t::cell_t; + using inter_t = samurai::Interval; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = samurai::CellArray; + + struct Data_t { + size_t level; + inter_t interval; + xt::xtensor_fixed> indices; + }; + + boost::mpi::communicator world; + + // For debug + std::ofstream logs; + logs.open( "log_" + std::to_string( _rank ) + ".dat", std::ofstream::app ); + logs << "# New load balancing (load_balancing_sfc)" << std::endl; + + auto & currentMesh = mesh[ Mesh_t::mesh_id_t::cells ]; + + // SFC key (used for implicit sorting through map mechanism) + std::map sfc_map; + + // FIXME: should a parameter of the function to adjust level ? + int sfc_max_level = currentMesh.max_level(); + + // boundaries of keys in current process + SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); + + size_t ninterval = 0; + samurai::for_each_interval( mesh, [&]( std::size_t level, const auto& inter, const auto& index ){ + + // get Logical coordinate or first cell + xt::xtensor_fixed> icell; + + // first element of interval + icell( 0 ) = inter.start; + for(int idim=0; idim> ijk; + for(size_t idim=0; idim( icell( idim ) ); + } + + sfc_map[ _sfc.template getKey( ijk ) ] = { level, inter, index }; + + ninterval ++; + }); + + assert( ninterval == sfc_map.size() ); + + // Key boundaries of current process + SFC_key_t interval[ 2 ] = { sfc_map.begin()->first, sfc_map.rbegin()->first }; + + logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; + + std::vector load_interval; + int my_load_i = static_cast( cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load_i, load_interval ); + + // compute load to transfer to neighbour rank-1, rank+1 + int neighbour_rank_prev = -1, neighbour_rank_next = -1; + int transfer_load_prev, transfer_load_next; + + // define neighbour processes for load-balancing, not geometrical neighbour ! + if( _rank > 0 ) { + neighbour_rank_prev = _rank - 1; + // transfer 50 % max of difference + transfer_load_prev = - ( my_load_i - load_interval[ neighbour_rank_prev ] ) * 0.5; + } + + if( _rank < _ndomains - 1 ) { + neighbour_rank_next = _rank + 1; + // transfer 50 % max of difference + transfer_load_next = - ( my_load_i - load_interval[ neighbour_rank_next ] ) * 0.5; + } + + logs << "Neighbour prev : " << neighbour_rank_prev << ", loads : " << transfer_load_prev << std::endl; + logs << "Neighbour next : " << neighbour_rank_next << ", loads : " << transfer_load_next << std::endl; + + // need send data to prev neighbour + if( neighbour_rank_prev >= 0 && transfer_load_prev != 0 ){ + + if( transfer_load_prev < 0 ){ + CellList_t cl_to_send; + + // give n-smallest morton keys to prev neighbour + size_t niter_send = 0; + for (auto iter = sfc_map.begin(); iter != sfc_map.end(); ++iter) { + if( transfer_load_prev < 0 && my_load_i > 0 ){ + cl_to_send[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); + my_load_i -= 1; + transfer_load_prev += 1; + niter_send ++; + }else{ + break; + } + } + + logs << "\t> Sending " << niter_send << " interval to " << neighbour_rank_prev << std::endl; + + CellArray_t ca_to_send = { cl_to_send, false }; + world.send( neighbour_rank_prev, 42, ca_to_send ); + mesh.remove( ca_to_send ); + + }else{ + // need recv + CellArray_t ca_to_rcv; + world.recv( neighbour_rank_prev, 42, ca_to_rcv ); + mesh.merge( ca_to_rcv ); + } + + } + + if( neighbour_rank_next > 0 && transfer_load_next != 0 ){ + + if( transfer_load_next < 0 ){ + CellList_t cl_to_send; + + // give n-smallest morton keys to prev neighbour + size_t niter_send = 0; + for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) { + if( transfer_load_next < 0 && my_load_i > 0 ){ + cl_to_send[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); + my_load_i -= 1; + transfer_load_next += 1; + niter_send ++; + }else{ + break; + } + } + + logs << "\t> Sending " << niter_send << " interval to " << neighbour_rank_next << std::endl; + + CellArray_t ca_to_send = { cl_to_send, false }; + world.send( neighbour_rank_next, 42, ca_to_send ); + mesh.remove( ca_to_send ); + + }else{ + // need recv + CellArray_t ca_to_rcv; + world.recv( neighbour_rank_next, 42, ca_to_rcv ); + mesh.merge( ca_to_rcv ); + } + + } + + + // update neighbour mesh - this should end up with the same result but .. + mesh.update_mesh_neighbour(); + + // update neighbour connectivity + auto requireNextIter = samurai::discover_neighbour( mesh ); + requireNextIter = samurai::discover_neighbour( mesh ); + + + } + +}; \ No newline at end of file From a5d13ba30cb365c5de144345476622a984d5bab6 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:33:16 +0100 Subject: [PATCH 016/170] add mrmesh constructeur for ca_type --- include/samurai/mr/mesh.hpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/samurai/mr/mesh.hpp b/include/samurai/mr/mesh.hpp index 2ab62fbb4..f5b0daf44 100644 --- a/include/samurai/mr/mesh.hpp +++ b/include/samurai/mr/mesh.hpp @@ -73,6 +73,7 @@ namespace samurai MRMesh() = default; MRMesh(const cl_type& cl, const self_type& ref_mesh); + MRMesh(const ca_type& ca, const self_type& ref_mesh); MRMesh(const cl_type& cl, std::size_t min_level, std::size_t max_level); MRMesh(const samurai::Box& b, std::size_t min_level, @@ -98,6 +99,12 @@ namespace samurai { } + template + inline MRMesh::MRMesh(const ca_type& ca, const self_type& ref_mesh) + : base_type(ca, ref_mesh) + { + } + template inline MRMesh::MRMesh(const cl_type& cl, std::size_t min_level, std::size_t max_level) : base_type(cl, min_level, max_level) From 1fdb5a1404252cf8feff10e35f4f11d72dc63268 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:38:52 +0100 Subject: [PATCH 017/170] remove load balancing from mesh class + add merge & remove functions --- include/samurai/mesh.hpp | 156 +++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 81 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 4f8387551..ea086bca2 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -104,6 +104,7 @@ namespace samurai bool is_periodic(std::size_t d) const; const std::array& periodicity() const; // std::vector& neighbouring_ranks(); + std::vector& mpi_neighbourhood(); void swap(Mesh_base& mesh) noexcept; @@ -132,12 +133,17 @@ namespace samurai void update_mesh_neighbour(); void to_stream(std::ostream& os) const; + + void merge( ca_type & lca ); + void remove( ca_type & lca ); + protected: using derived_type = D; Mesh_base() = default; // cppcheck-suppress uninitMemberVar Mesh_base(const cl_type& cl, const self_type& ref_mesh); + Mesh_base( const ca_type &ca, const self_type & ref_mesh); Mesh_base(const cl_type& cl, std::size_t min_level, std::size_t max_level); Mesh_base(const samurai::Box& b, std::size_t start_level, @@ -233,7 +239,6 @@ namespace samurai #ifdef SAMURAI_WITH_MPI partition_mesh(start_level, b); - // load_balancing(); #else this->m_cells[mesh_id_t::cells][start_level] = {start_level, b, approx_box_tol, scaling_factor_}; #endif @@ -321,6 +326,24 @@ namespace samurai set_scaling_factor(ref_mesh.scaling_factor()); } + template + inline Mesh_base::Mesh_base(const ca_type& ca, const self_type& ref_mesh) + : m_domain(ref_mesh.m_domain) + , m_min_level(ref_mesh.m_min_level) + , m_max_level(ref_mesh.m_max_level) + , m_periodic(ref_mesh.m_periodic) + , m_mpi_neighbourhood(ref_mesh.m_mpi_neighbourhood) + + { + m_cells[mesh_id_t::cells] = ca; + + update_mesh_neighbour(); + construct_subdomain(); + construct_union(); + update_sub_mesh(); + renumbering(); + } + template inline auto Mesh_base::cells() -> mesh_t& { @@ -647,8 +670,7 @@ namespace samurai } template - void Mesh_base::partition_mesh([[maybe_unused]] std::size_t start_level, [[maybe_unused]] const Box& global_box) - { + void Mesh_base::partition_mesh([[maybe_unused]] std::size_t start_level, [[maybe_unused]] const Box& global_box) { #ifdef SAMURAI_WITH_MPI mpi::communicator world; auto rank = world.rank(); @@ -715,6 +737,7 @@ namespace samurai this->m_cells[mesh_id_t::cells][start_level] = {start_level, subdomain_box}; */ + /** delete for git rebase lcl_type subdomain_cells(start_level, m_domain.origin_point(), m_domain.scaling_factor()); auto subdomain_nb_intervals = m_domain.nb_intervals() / static_cast(size); auto subdomain_start = static_cast(rank) * subdomain_nb_intervals; @@ -732,6 +755,9 @@ namespace samurai }); this->m_cells[mesh_id_t::cells][start_level] = subdomain_cells; + // end comment for git rebase + **/ + this->m_cells[mesh_id_t::cells][start_level] = {start_level, subdomain_box}; m_mpi_neighbourhood.reserve(static_cast(size) - 1); for (int ir = 0; ir < size; ++ir) @@ -756,100 +782,68 @@ namespace samurai // return neighbour_rank; // }; - // static_nested_loop( - // [&](auto& shift) - // { - // if (xt::any(shift)) - // { - // for (std::size_t d = 0; d < dim; ++d) - // { - // if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) - // { - // return; - // } - // } - // m_mpi_neighbourhood.push_back(neighbour(shift)); - // } - // }); - + static_nested_loop( + [&](auto& shift) + { + if (xt::any(shift)) + { + for (std::size_t d = 0; d < dim; ++d) + { + if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) + { + return; + } + } + m_mpi_neighbourhood.push_back(neighbour(shift)); + } + }); + #endif } template - void Mesh_base::load_balancing() - { -#ifdef SAMURAI_WITH_MPI - mpi::communicator world; - auto rank = world.rank(); + void Mesh_base::merge( ca_type & lca ) { + // merge received cells - std::size_t load = nb_cells(mesh_id_t::cells); - std::vector loads; + auto & refmesh = this->m_cells[ mesh_id_t::cells ]; - std::vector load_fluxes(m_mpi_neighbourhood.size(), 0); + auto minlevel = std::min( refmesh.min_level(), lca.min_level() ); + auto maxlevel = std::max( refmesh.max_level(), lca.max_level() ); - const std::size_t n_iterations = 1; + cl_type cl; + for( size_t ilvl=minlevel; ilvl<=maxlevel; ++ilvl ) { - for (std::size_t k = 0; k < n_iterations; ++k) - { - world.barrier(); - if (rank == 0) - { - std::cout << "---------------- k = " << k << " ----------------" << std::endl; - } - mpi::all_gather(world, load, loads); + auto un = samurai::union_( refmesh[ ilvl ], lca[ ilvl ] ); - std::vector nb_neighbours; - mpi::all_gather(world, m_mpi_neighbourhood.size(), nb_neighbours); + un([&]( auto & interval, auto & indices ) { + cl[ ilvl ][ indices ].add_interval( interval ); + }); + } - double load_np1 = static_cast(load); - for (std::size_t i_rank = 0; i_rank < m_mpi_neighbourhood.size(); ++i_rank) - { - auto neighbour = m_mpi_neighbourhood[i_rank]; + refmesh = { cl, false }; + } - auto neighbour_load = loads[static_cast(neighbour.rank)]; - int neighbour_load_minus_my_load; - if (load < neighbour_load) - { - neighbour_load_minus_my_load = static_cast(neighbour_load - load); - } - else - { - neighbour_load_minus_my_load = -static_cast(load - neighbour_load); - } - double weight = 1. / std::max(m_mpi_neighbourhood.size(), nb_neighbours[neighbour.rank]); - load_fluxes[i_rank] = weight * neighbour_load_minus_my_load; - load_np1 += load_fluxes[i_rank]; - } - load_np1 = floor(load_np1); + template + void Mesh_base::remove( ca_type & lca) { + auto & refmesh = this->m_cells[ mesh_id_t::cells ]; - load_transfer(load_fluxes); + // remove cells + cl_type cl; + size_t diff_ncells = 0; + for( int ilvl=refmesh.min_level(); ilvl<=refmesh.max_level(); ++ilvl ) { - std::cout << rank << ": load = " << load << ", load_np1 = " << load_np1 << std::endl; + auto diff = samurai::difference( refmesh[ ilvl ], lca[ ilvl ] ); - load = static_cast(load_np1); - } -#endif - } + diff([&]( auto & interval, auto & index ) { + cl[ ilvl ][ index ].add_interval( interval ); + diff_ncells += interval.size(); + }); - template - void Mesh_base::load_transfer([[maybe_unused]] const std::vector& load_fluxes) - { -#ifdef SAMURAI_WITH_MPI - mpi::communicator world; - std::cout << world.rank() << ": "; - for (std::size_t i_rank = 0; i_rank < m_mpi_neighbourhood.size(); ++i_rank) - { - auto neighbour = m_mpi_neighbourhood[i_rank]; - if (load_fluxes[i_rank] < 0) // must tranfer load to the neighbour - { - } - else if (load_fluxes[i_rank] > 0) // must receive load from the neighbour - { - } - std::cout << "--> " << neighbour.rank << ": " << load_fluxes[i_rank] << ", "; } - std::cout << std::endl; -#endif + + // new mesh for current process + refmesh = { cl, false }; + } template From a184e83706e5c2e120173e081107d52ed0acd84a Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:39:17 +0100 Subject: [PATCH 018/170] add gravity base load balancing for cells --- .../samurai/load_balancing_diffusion_cell.hpp | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 include/samurai/load_balancing_diffusion_cell.hpp diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp new file mode 100644 index 000000000..92bccb68f --- /dev/null +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -0,0 +1,262 @@ +#pragma once + +#include +#include "load_balancing.hpp" + +template +class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer> { + + using Coord_t = xt::xtensor_fixed>; + + private: + int _ndomains; + int _rank; + + template + double getSurfaceOrVolume( Mesh_t & mesh ) const { + + double s_ = 0.; + for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ + + double s = 1.; + + for(int idim=0; idim + void load_balance_impl( Mesh_t & mesh ){ + + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = samurai::CellArray; + using Cell_t = typename Mesh_t::cell_t; + using mesh_id_t = typename Mesh_t::mesh_id_t; + using Coord_t = xt::xtensor_fixed>; + + boost::mpi::communicator world; + + // For debug purpose + std::ofstream logs; + logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs << "# New load balancing" << std::endl; + + // give access to rank & mesh of neighbour + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + + size_t n_neighbours = neighbourhood.size(); + + std::vector loads; + double my_load = static_cast( cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load, loads ); + + // get the load to neighbours (geometrical neighbour) + std::vector fluxes = samurai::cmptFluxes( mesh ); + + { + logs << "load : " << my_load << std::endl; + logs << "nneighbours : " << n_neighbours << std::endl; + logs << "neighbours : "; + for( size_t in=0; in( mesh ); + + // compute some point of reference in mesh and interval-based interface + // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = _cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); + logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; + + // std::vector barycenter_interface_neighbours( n_neighbours ); + std::vector barycenter_neighbours( n_neighbours ); + std::vector sv( n_neighbours ); + + for(size_t nbi=0; nbi( interface[ nbi ] ); + barycenter_neighbours[ nbi ] = _cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); + + sv[ nbi ] = getSurfaceOrVolume( neighbourhood[ nbi ].mesh ); + + // debug + auto s_ = fmt::format( "Barycenter neighbour # {}: ({}, {})", neighbourhood[ nbi ].rank, + barycenter_neighbours[ nbi ]( 0 ), + barycenter_neighbours[ nbi ]( 1 ) ); + + logs << s_ << std::endl; + } + + struct Data { + Cell_t cell; + int rank; + }; + + // build map of interval that needs to be sent. Warning, it does not work with classical std::map !! + std::multimap repartition; + + constexpr auto fdist = samurai::Distance_t::GRAVITY; + + double currentSV = getSurfaceOrVolume( mesh ); + + for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ + + auto cc = cell.center(); + + // process that might get the interval + int winner_id = -1; + + // double winner_dist = std::numeric_limits::max(); + // double winner_dist = samurai::getDistance( cell, barycenter ) / loads[ world.rank() ]; + + double coeff_current = currentSV / loads[ world.rank() ]; + double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; + + // select the neighbour + for( size_t ni=0; ni= 0 ) continue; // skip neighbour that will recv + + // this might fix ilots but require neighbour update + // double dist = std::min( distance_inf( mid_point, barycenter_interface_neighbours[ ni ] ), + // distance_inf( mid_point, barycenter_neighbours[ ni ] ) ); + + // double dist = samurai::getDistance( cell, barycenter_interface_neighbours[ ni ] ) / loads[ neighbour_rank ]; + double coeff = sv[ ni ] / loads[ neighbour_rank ]; // / sv[ ni ]; + double dist = samurai::getDistance( cell.center(), barycenter_neighbours[ ni ] ) * coeff; + + // double dist = std::max( d1, d2 ); + + if( dist < winner_dist ){ + winner_id = ni; + winner_dist = dist; + } + + } + + if( winner_id >= 0 ){ + + if( repartition.find( winner_dist ) != repartition.find( winner_dist ) ){ + std::cerr << "\t> WARNING: Key conflict for std::map !" << std::endl; + } + + repartition.insert ( std::pair( winner_dist, Data { cell, static_cast( winner_id ) } ) ); + + } + + }); + + + std::vector cl_to_send( n_neighbours ); + std::vector ca_to_send( n_neighbours ); + + // distribute intervals based on ordered distance to neighbours + // rank is not the rank but the offset in the neighbourhood + std::vector given_ ( n_neighbours, 0 ); + // for( auto & it : repartition ){ + for( auto it = repartition.begin(); it != repartition.end(); it++ ){ + + auto nrank = neighbourhood[ it->second.rank ].rank; + auto lvl = it->second.cell.level; + + // shouldn't we give it to the second closest neighbour ?! + if( given_[ it->second.rank ] + 1 <= ( - fluxes[ it->second.rank ] ) ){ + + if constexpr ( dim == 3 ) { + auto i = it->second.cell.indices[ 0 ]; + auto j = it->second.cell.indices[ 1 ]; + auto k = it->second.cell.indices[ 2 ]; + cl_to_send[ it->second.rank ][ it->second.cell.level ][ { j, k } ].add_point( i ); + }else{ + auto i = it->second.cell.indices[ 0 ]; + auto j = it->second.cell.indices[ 1 ]; + cl_to_send[ it->second.rank ][ it->second.cell.level ][ { j } ].add_point( i ); + } + + given_[ it->second.rank ] += 1; + + } + + } + + for(size_t nbi=0; nbi Number of cells to send to process # " << neighbourhood[ nbi ].rank + << " : " << ca_to_send[ nbi ].nb_cells() << std::endl; + } + + // actual data transfer occurs here + for(size_t ni=0; ni 0 ) { // receive data + samurai::CellArray to_rcv; + world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); + + logs << "Receiving data from # " << neighbourhood[ ni ].rank + << ", nbCells : " << to_rcv.nb_cells() << std::endl; + + logs << "Merging cells, before : " << mesh.nb_cells( mesh_id_t::cells ) << std::endl; + mesh.merge( to_rcv ); + logs << "Merging cells, after : " << mesh.nb_cells( mesh_id_t::cells ) << std::endl; + + }else{ // send data to + world.send( neighbourhood[ ni ].rank, 42, ca_to_send[ ni ] ); + + logs << "Sending data to # " << neighbourhood[ ni ].rank + << ", nbCells : " << ca_to_send[ ni ].nb_cells() << std::endl; + + logs << "Removing cells, before : " << mesh.nb_cells( mesh_id_t::cells ) << std::endl; + + mesh.remove( ca_to_send[ ni ] ); + + logs << "Removing cells, after : " << mesh.nb_cells( mesh_id_t::cells ) << std::endl; + + } + } + + // update neighbourhood + // send current process mesh to neighbour and get neighbour mesh + mesh.update_mesh_neighbour(); + + // discover neighbours, since it might have changed + bool requireNextIter = true; + + // while( requireNextIter ){ + requireNextIter = samurai::discover_neighbour( mesh ); + requireNextIter = samurai::discover_neighbour( mesh ); + // } + + } + +}; \ No newline at end of file From 2c4a867cfad0a87a4eb9b7a769ea5335cfafc9af Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:39:37 +0100 Subject: [PATCH 019/170] add graph based with interface propagation load balancing class --- .../load_balancing_diffusion_interval.hpp | 410 ++++++++++++++++++ 1 file changed, 410 insertions(+) create mode 100644 include/samurai/load_balancing_diffusion_interval.hpp diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp new file mode 100644 index 000000000..c20c85c8a --- /dev/null +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -0,0 +1,410 @@ +#pragma once + +#include +#include "load_balancing.hpp" + +template +class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer> { + + using Coord_t = xt::xtensor_fixed>; + + private: + int _ndomains; + int _rank; + + /** + * Compute the barycenter of the current domain based on mid-center of intervals. + * Weighted by level. + * + */ + template + Coord_t _cmpIntervalBarycenter( Mesh_t & mesh ) { // compute barycenter of current domains ( here, all domains since with simulate multiple MPI domains) + + Coord_t bary; + bary.fill( 0. ); + + double wght_tot = 0.; + samurai::for_each_interval( mesh, + [&]( std::size_t level, const auto& interval, const auto& index ) { + + // [OPTIMIZATION] precompute weight as array + // double wght = 1. / ( 1 << ( mesh.max_level() - level ) ); + constexpr double wght = 1.; + + Coord_t mid = _getIntervalMidPoint( level, interval, index ); + + for(int idim=0; idim + inline Coord_t _getIntervalMidPoint( size_t level, const Interval_t & interval, + const Index_t & index ) const { + Coord_t mid; + double csize = samurai::cell_length( level ); + + mid( 0 ) = ( (interval.end - interval.start) * 0.5 + interval.start ) * csize ; + for( int idim=0; idim + void load_balance_impl( Mesh_t & mesh ){ + + using interval_t = typename Mesh_t::interval_t; + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = samurai::CellArray; + using mesh_id_t = typename Mesh_t::mesh_id_t; + using Coord_t = xt::xtensor_fixed>; + using Cell_t = typename Mesh_t::cell_t; + + boost::mpi::communicator world; + + // For debug + std::ofstream logs; + logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs << "# New load balancing" << std::endl; + + // compute fluxes in terms of number of intervals to transfer/receive + std::vector fluxes = samurai::cmptFluxes( mesh ); + std::vector new_fluxes( fluxes ); + + // get loads from everyone + std::vector loads; + int my_load = static_cast( cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load, loads ); + + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + size_t n_neighbours = neighbourhood.size(); + + { // some debug info + logs << "load : " << my_load << std::endl; + logs << "nneighbours : " << n_neighbours << std::endl; + logs << "neighbours : "; + for( size_t in=0; in cl_to_send( n_neighbours ); + + bool balancing_done = false; + while( ! balancing_done ){ + + // select neighbour to which we needs to sent the more load + int requester = -1; + int requested_load = 0; + for(size_t nbi=0; nbi= 0 ) continue; + if( - new_fluxes[ nbi ] > requested_load ){ + requested_load = - new_fluxes[ nbi ]; + requester = static_cast( nbi ); + } + } + + if( requester < 0 ){ + { // debug + logs << "No more neighbour found " << std::endl; + } + + balancing_done = true; + break; + } + + { // debug + logs << "Requester neighbour : " << neighbourhood[ requester ].rank << ", fluxes : " << requested_load << std::endl; + } + + // select interval for this neighbour by moving in cartesian direction by one + auto interface = cmptInterface( mesh, neighbourhood[ requester ].mesh ); + + { // check emptyness of interface, if it is empty, then set fluxes for this neighbour to 0 + size_t nelement = 0; + samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ + nelement += 1; + }); + + if( nelement == 0 ){ + new_fluxes[ requester ] = 0; + + { // debug + logs << "Requester neighbour, no interface found, set fluxes to " << new_fluxes[ requester ] << std::endl; + } + }else{ + { // debug + logs << "Requester neighbour, interface " << nelement << " intervals " << std::endl; + } + } + } + + // go through interval on the interface and add as much as possible + // skip this to add the whole interface + CellList_t cl_for_neighbour; + + size_t nbIntervalAdded = 0; + samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ + + if( new_fluxes[ requester ] < 0 ){ + cl_for_neighbour[ level ][ index ].add_interval( interval ); + // new_fluxes[ requester ] += 1; // here interval load == 1, no weight; + nbIntervalAdded += 1; + } + + }); + + new_fluxes[ requester ] += nbIntervalAdded; + + { // remove interval from current process mesh and add it to neighbour mesh local copy ! + CellArray_t ca_for_neighbour = { cl_for_neighbour, false }; + mesh.remove( ca_for_neighbour ); + neighbourhood[ requester ].mesh.merge( ca_for_neighbour ); + + // update gobal list that will be sent to neighbour process + samurai::for_each_interval( ca_for_neighbour, [&]( std::size_t level, const auto & interval, const auto & index ){ + cl_to_send[ requester ][ level ][ index ].add_interval( interval ); + }); + } + + { // debug + logs << "New flux for this neighbour : " << new_fluxes[ requester ] << std::endl; + } + + } + + // at this point local mesh of neighbour are modified ( technically it should match what would results from send) + // and local mesh has been modified + for(size_t ni=0; ni 0 ) { // receive data + samurai::CellArray to_rcv; + world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); + mesh.merge( to_rcv ); + }else{ // send data to + CellArray_t to_send = { cl_to_send[ ni ], false }; + world.send( neighbourhood[ ni ].rank, 42, to_send ); + } + } + + // update neighbour mesh - this should end up with the same result but .. + mesh.update_mesh_neighbour(); + + // update neighbour connectivity + auto requireNextIter = samurai::discover_neighbour( mesh ); + requireNextIter = samurai::discover_neighbour( mesh ); + + } + + template + void load_balance_impl2( Mesh_t & mesh ){ + + using interval_t = typename Mesh_t::interval_t; + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = samurai::CellArray; + using mesh_id_t = typename Mesh_t::mesh_id_t; + using Coord_t = xt::xtensor_fixed>; + + boost::mpi::communicator world; + + std::ofstream logs; + // DEBUG + logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs << "# New load balancing" << std::endl; + + std::string smt = "# " + std::to_string( _rank ) + " [Diffusion_LoadBalancer_interval]::load_balance_impl "; + SAMURAI_TRACE( smt + "Entering function ..." ); + + // give access to rank & mesh of neighbour + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + size_t n_neighbours = neighbourhood.size(); + + // get the load to neighbours (geometrical neighbour) + std::vector fluxes = samurai::cmptFluxes( mesh ); + + { + logs << "load : " << cmptLoad( mesh ) << std::endl; + logs << "nneighbours : " << n_neighbours << std::endl; + logs << "neighbours : "; + for( size_t in=0; in( mesh ); + + // compute some point of reference in mesh and interval-based interface + // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = _cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); + logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; + + std::vector invloads; + double my_load = static_cast( cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load, invloads ); + for(size_t il=0; il barycenter_interface_neighbours( n_neighbours ); + std::vector barycenter_neighbours( n_neighbours ); + + for(size_t nbi=0; nbi( interface[ nbi ] ); + barycenter_neighbours[ nbi ] = _cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); + + // debug + // auto s_ = fmt::format( "Barycenter neighbour : ({}, {})", + // barycenter_interface_neighbours[ nbi ]( 0 ), + // barycenter_interface_neighbours[ nbi ]( 1 ) ); + + // logs << s_ << std::endl; + } + + struct Data { + size_t level; + interval_t interval; + xt::xtensor_fixed> indices; + int rank; + }; + + // build map of interval that needs to be sent + std::multimap repartition; + + constexpr auto fdist = samurai::Distance_t::GRAVITY; + + for_each_interval( mesh[ Mesh_t::mesh_id_t::cells ], + [&]( std::size_t level, const auto& interval, const auto& index ){ + + // cartesian coordinates + auto mid_point = _getIntervalMidPoint( level, interval, index ); + + // process that might get the interval + int winner_id = -1; + // double winner_dist = std::numeric_limits::max(); + double winner_dist = samurai::getDistance( mid_point, barycenter ) * invloads[ _rank ]; + + // select the neighbour + for( size_t ni=0; ni= 0 ) continue; // skip neighbour that will recv + + // this might fix ilots but require neighbour update + // double dist = std::min( distance_inf( mid_point, barycenter_interface_neighbours[ ni ] ), + // distance_inf( mid_point, barycenter_neighbours[ ni ] ) ); + + double dist = samurai::getDistance( mid_point, barycenter_neighbours[ ni ] ) * invloads[ neighbour_rank ] ; + // double dist = samurai::distance_inf( mid_point, barycenter_interface_neighbours[ ni ] ); + + if( dist < winner_dist ){ + winner_id = ni; + winner_dist = dist; + } + + } + + if( winner_id >= 0 ){ + + if( repartition.find( winner_dist ) != repartition.find( winner_dist ) ){ + std::cerr << "\t> WARNING: Key conflict for std::map !" << std::endl; + } + + repartition.insert ( std::pair( winner_dist, Data { level, interval, index, static_cast( winner_id ) } ) ); + + } + + }); + + std::vector cl_to_send( n_neighbours ); + std::vector ca_to_send( n_neighbours ); + + // distribute intervals based on ordered distance to neighbours + // rank is not the rank but the offset in the neighbourhood + std::vector given_ ( n_neighbours, 0 ); + // for( auto & it : repartition ){ + for( auto it = repartition.begin(); it != repartition.end(); it++ ){ + + auto nrank = neighbourhood[ it->second.rank ].rank; + auto lvl = it->second.level; + + logs << fmt::format("\t> Interval {} --> to rank {} ( level : {} , dist : {}", it->second.interval, nrank, lvl, it->first) << std::endl; + + // shouldn't we give it to the second closest neighbour ?! + if( given_[ it->second.rank ] + it->second.interval.size() <= ( - fluxes[ it->second.rank ] ) ){ + cl_to_send[ it->second.rank ][ it->second.level ][ it->second.indices ].add_interval( it->second.interval ); + given_[ it->second.rank ] += it->second.interval.size(); + } + + } + + for(size_t nbi=0; nbi Number of cells to send to process # " << neighbourhood[ nbi ].rank + << " : " << ca_to_send[ nbi ].nb_cells() << std::endl; + } + + // actual data transfer occurs here + for(size_t ni=0; ni 0 ) { // receive data + samurai::CellArray to_rcv; + world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); + mesh.merge( to_rcv ); + }else{ // send data to + world.send( neighbourhood[ ni ].rank, 42, ca_to_send[ ni ] ); + mesh.remove( ca_to_send[ ni ] ); + } + } + + // update neighbour mesh + mesh.update_mesh_neighbour(); + } + +}; \ No newline at end of file From ad3cbfa3421c45f5ac2f8308f33c868b05ace4fc Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:40:43 +0100 Subject: [PATCH 020/170] add load balancing demo --- demos/loadbalancing/CMakeLists.txt | 7 +- demos/loadbalancing/circle.cpp | 439 +++++++++++++++++++++++++++++ demos/loadbalancing/partition.cpp | 286 +++++++++++++++++-- 3 files changed, 714 insertions(+), 18 deletions(-) create mode 100644 demos/loadbalancing/circle.cpp diff --git a/demos/loadbalancing/CMakeLists.txt b/demos/loadbalancing/CMakeLists.txt index 8227a998a..b62de8e28 100644 --- a/demos/loadbalancing/CMakeLists.txt +++ b/demos/loadbalancing/CMakeLists.txt @@ -1,2 +1,5 @@ -add_executable(test-partition partition.cpp) -target_link_libraries(test-partition samurai CLI11::CLI11) +add_executable(test-simple-2d partition.cpp) +target_link_libraries(test-simple-2d samurai CLI11::CLI11) + +add_executable(test-circle-2d circle.cpp) +target_link_libraries(test-circle-2d samurai CLI11::CLI11) \ No newline at end of file diff --git a/demos/loadbalancing/circle.cpp b/demos/loadbalancing/circle.cpp new file mode 100644 index 000000000..946b40fb4 --- /dev/null +++ b/demos/loadbalancing/circle.cpp @@ -0,0 +1,439 @@ +#include + +// CLI +#include + +// samurai +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +template +auto init(Mesh& mesh, const double radius, const double x_center, const double y_center ) +{ + auto u = samurai::make_field("u", mesh); + + samurai::for_each_cell( mesh, [&](auto& cell) { + auto center = cell.center(); + double rad = 0.; + double z_center = 0.5; + + if constexpr ( dim == 3 ){ + rad = ((center[0] - x_center) * (center[0] - x_center) + + (center[1] - y_center) * (center[1] - y_center) + + (center[2] - z_center) * (center[2] - z_center) ); + }else{ + rad = ((center[0] - x_center) * (center[0] - x_center) + + (center[1] - y_center) * (center[1] - y_center) ); + } + if ( rad <= radius * radius) { + u[cell] = 1; + } else { + u[cell] = 0; + } + }); + + return u; +} + +template +auto initMultiCircles(Mesh& mesh, const std::vector &radius, const std::vector & centers ) +{ + auto u = samurai::make_field("u", mesh); + + samurai::for_each_cell( mesh, [&](auto& cell) { + auto cc = cell.center(); + + for(size_t icerc=0; icerc> minCorner = {0., 0., 0.}; + xt::xtensor_fixed> maxCorner = {1., 1., 1.}; + std::string filename = "ld_mesh_init"; + + double radius = 0.2, x_center = 0.5, y_center = 0.5; + int niterBench = 1; + + CLI::App app{"Load balancing test"}; + app.add_option("--ndomains", ndomains, "Number of desired domains")->capture_default_str()->group("Simulation parameters"); + app.add_option("--nbIterLB", nbIterLoadBalancing, "Number of desired lb iteration") + ->capture_default_str()->group("Simulation parameters"); + app.add_option("--niterBench", niterBench, "Number of iteration for bench")->capture_default_str()->group("Simulation parameters"); + app.add_option("--min-corner", minCorner, "The min corner of the box")->capture_default_str()->group("Simulation parameters"); + app.add_option("--max-corner", maxCorner, "The max corner of the box")->capture_default_str()->group("Simulation parameters"); + app.add_option("--min-level", minLevel, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--max-level", maxLevel, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-eps", mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") + ->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-reg", mr_regularity,"The regularity criteria used by the multiresolution to " + "adapt the mesh")->capture_default_str()->group("Multiresolution"); + app.add_option("--filename", filename, "File name prefix")->capture_default_str()->group("Ouput"); + app.add_option("--radius", radius, "Bubble radius")->capture_default_str()->group("Simulation parameters"); + app.add_option("--xcenter", x_center, "Bubble x-axis center")->capture_default_str()->group("Simulation parameters"); + app.add_option("--ycenter", y_center, "Bubble y-axis center")->capture_default_str()->group("Simulation parameters"); + + CLI11_PARSE(app, argc, argv); + + double cfl = 0.5; + double dt = cfl / (1 << maxLevel); + + // Convection operator + samurai::VelocityVector velocity; + velocity.fill(1); + velocity(1) = -1; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using mesh_id_t = typename Mesh_t::mesh_id_t; + + boost::mpi::communicator world; + + if( world.rank() == 0 ) std::cerr << "\t> Testing Diffusion_LoadBalancer_interval " << std::endl; + + { // load balancing cells by cells using gravity loadbalancer + + Timers myTimers; + + /** + * Initialize AMR mesh with spherical buble + */ + myTimers.start("InitMesh"); + + samurai::Box box( minCorner, maxCorner ); + + Mesh_t mesh(box, minLevel, maxLevel); + + std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; + + auto u = init( mesh, radius, x_center, y_center ); + + // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + + samurai::make_bc(u, 0.); + auto mradapt = samurai::make_MRAdapt( u ); + mradapt( mr_epsilon, mr_regularity ); + + myTimers.stop("InitMesh"); + + samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + + myTimers.start( "balance_DIF_inter" ); + Diffusion_LoadBalancer_interval _diff_lb; + for(size_t lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); + + samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + + auto conv = samurai::make_convection(velocity); + + // myTimers.start( "update_ghost_mr" ); + // for(int i=0; i("unp1", newmesh); + + myTimers.start( "upwind" ); + for(int i=0; i Testing Diffusion_LoadBalancer_cell " << std::endl; + + { // load balancing cells by cells using gravity loadbalancer + + Timers myTimers; + + /** + * Initialize AMR mesh with spherical buble + */ + myTimers.start("InitMesh"); + + samurai::Box box( minCorner, maxCorner ); + + Mesh_t mesh(box, minLevel, maxLevel); + + std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; + + auto u = init( mesh, radius, x_center, y_center ); + // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + + samurai::make_bc(u, 0.); + auto mradapt = samurai::make_MRAdapt( u ); + mradapt( mr_epsilon, mr_regularity ); + + myTimers.stop("InitMesh"); + + samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + + myTimers.start( "balance_DIF_cell" ); + Diffusion_LoadBalancer_cell _diff_lb_c; + for(int lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); + + samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + + auto conv = samurai::make_convection(velocity); + + // myTimers.start( "update_ghost_mr" ); + // for(int i=0; i("unp1", newmesh); + + myTimers.start( "upwind" ); + for(int i=0; i Testing Morton_LoadBalancer_interval " << std::endl; + + { // load balancing cells by cells using gravity loadbalancer + + Timers myTimers; + + /** + * Initialize AMR mesh with spherical buble + */ + myTimers.start("InitMesh"); + + samurai::Box box( minCorner, maxCorner ); + + Mesh_t mesh(box, minLevel, maxLevel); + + std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; + + auto u = init( mesh, radius, x_center, y_center ); + // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + + samurai::make_bc(u, 0.); + auto mradapt = samurai::make_MRAdapt( u ); + mradapt( mr_epsilon, mr_regularity ); + + myTimers.stop("InitMesh"); + + samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + + myTimers.start( "balance_Morton_cell" ); + SFC_LoadBalancer_interval loadb_morton; + for(int lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); + + samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + + auto conv = samurai::make_convection(velocity); + + // myTimers.start( "update_ghost_mr" ); + // for(int i=0; i("unp1", newmesh); + + myTimers.start( "upwind" ); + for(int i=0; i Testing Morton_LoadBalancer_interval " << std::endl; + + { // load balancing cells by cells using gravity loadbalancer + + Timers myTimers; + + /** + * Initialize AMR mesh with spherical buble + */ + myTimers.start("InitMesh"); + + samurai::Box box( minCorner, maxCorner ); + + Mesh_t mesh(box, minLevel, maxLevel); + + std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; + + auto u = init( mesh, radius, x_center, y_center ); + // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + + samurai::make_bc(u, 0.); + auto mradapt = samurai::make_MRAdapt( u ); + mradapt( mr_epsilon, mr_regularity ); + + myTimers.stop("InitMesh"); + + samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + + myTimers.start( "balance_Hilbert_cell" ); + SFC_LoadBalancer_interval loadb_hilbert; + for(int lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); + + samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + + auto conv = samurai::make_convection(velocity); + + // myTimers.start( "update_ghost_mr" ); + // for(int i=0; i("unp1", newmesh); + + myTimers.start( "upwind" ); + for(int i=0; i #include #include + +#include + +#include #include #include #include -#include -#include +#include +#include + +#include +#include + +template +auto init(Mesh& mesh, const double radius, const double x_center, const double y_center ) +{ + auto u = samurai::make_field("u", mesh); + + samurai::for_each_cell( mesh, [&](auto& cell) { + auto center = cell.center(); + double rad = ((center[0] - x_center) * (center[0] - x_center) + + (center[1] - y_center) * (center[1] - y_center) ); + if ( rad <= radius * radius) { + u[cell] = 1; + } else { + u[cell] = 0; + } + }); + + return u; +} + +template +void detectInterface( std::vector & meshes, std::vector & graph ){ + + std::cerr << "\t> [detectInterface] Number of mesh : " << meshes.size() << std::endl; + + for(int irank=0; irank Processing domain # " << irank << std::endl; + + for( int nbi=0; nbi Neighbour process # " << neighbour_rank << std::endl; + + for( int idim=0; idim<2*dim; ++idim ){ + + xt::xtensor_fixed> stencil; + stencil.fill( 0 ); + + stencil[ idim / 2 ] = 1 - ( idim % 2 ) * 2; + + std::cerr << "\t\t\t> Stencil {" << stencil[ 0 ] << ", " << stencil[ 1 ] << "}" << std::endl; + + size_t nbCellsIntersection = 0, nbIntervalIntersection = 0; + int minlevel = meshes[ neighbour_rank ].min_level(); + int maxlevel = meshes[ neighbour_rank ].max_level(); + + for( int level=minlevel; level<=maxlevel; ++level ) { + + // for each level look at level -1 / 0 / +1 + int minlevel_proj = meshes[ irank ].min_level(); + int maxlevel_proj = meshes[ irank ].max_level(); + for(size_t projlevel=std::max(minlevel_proj, level-1); projlevel<=std::min(maxlevel_proj, level+1); ++projlevel ){ + std::cerr << "\t\t\t> Looking intersection " << level << " onto level " << projlevel << std::endl; + + // intersection avec myself + // translate neighbour from dir (hopefully to current) + auto set = translate( meshes[ neighbour_rank ][ level ], stencil ); + auto intersect = intersection( set, meshes[ irank ][ projlevel ] ).on( projlevel ); + + size_t nbInter_ = 0, nbCells_ = 0; + intersect([&]( auto & i, auto & index ) { + nbInter_ += 1; + nbCells_ += i.size(); + }); + + // we get more interval / cells, than wanted because neighbour has bigger cells + if( nbInter_ > 0 && projlevel > level ){ + std::cerr << "\t\t\t> Too much cell selected ... " << std::endl; + auto set_ = translate( meshes[ irank ][ projlevel ], stencil ); + auto diff_ = difference( intersect, set_ ); + + nbInter_ = 0; + nbCells_ = 0; + diff_([&]( auto & i, auto & index ) { + nbInter_ += 1; + nbCells_ += i.size(); + }); + + } + + std::cerr << "\t\t\t> nbInter_ : " << nbInter_ << ", nbCells_ " << nbCells_ << std::endl; + + nbCellsIntersection = nbCells_ ; + nbIntervalIntersection = nbInter_ ; + + } + + } + std::cerr << "\t\t> Total Ncells intersected " << nbCellsIntersection << ", nb interval : " << nbIntervalIntersection << std::endl; + + } + + } + + } + +} int main( int argc, char * argv[] ){ samurai::initialize(argc, argv); Timers myTimers; - constexpr std::size_t dim = 2; + constexpr int dim = 2; + int ndomains = 1; std::size_t minLevel = 3, maxLevel = 8; - double mr_regularity = 1.0, mr_epsilon = 1e-4; + double mr_regularity = 1.0, mr_epsilon = 2.e-4; xt::xtensor_fixed> minCorner = {0., 0.}; xt::xtensor_fixed> maxCorner = {1., 1.}; - std::string filename = "load_balancing"; + std::string filename = "ld_mesh_init"; + + double radius = 0.2, x_center = 0.5, y_center = 0.5; CLI::App app{"Load balancing test"}; + app.add_option("--ndomains", ndomains, "Number of desired domains")->capture_default_str()->group("Simulation parameters"); app.add_option("--min-corner", minCorner, "The min corner of the box")->capture_default_str()->group("Simulation parameters"); app.add_option("--max-corner", maxCorner, "The max corner of the box")->capture_default_str()->group("Simulation parameters"); app.add_option("--min-level", minLevel, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); @@ -35,27 +144,172 @@ int main( int argc, char * argv[] ){ app.add_option("--mr-reg", mr_regularity,"The regularity criteria used by the multiresolution to " "adapt the mesh")->capture_default_str()->group("Multiresolution"); app.add_option("--filename", filename, "File name prefix")->capture_default_str()->group("Ouput"); + app.add_option("--radius", radius, "Bubble radius")->capture_default_str()->group("Simulation parameters"); + app.add_option("--xcenter", x_center, "Bubble x-axis center")->capture_default_str()->group("Simulation parameters"); + app.add_option("--ycenter", y_center, "Bubble y-axis center")->capture_default_str()->group("Simulation parameters"); CLI11_PARSE(app, argc, argv); - std::size_t start_level = minLevel; + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; + /** + * Initialize AMR mesh with spherical buble + */ myTimers.start("InitMesh"); - samurai::Box box( minCorner, maxCorner ); - samurai::CellArray mesh; + // samurai::Box box( minCorner, maxCorner ); - mesh[start_level] = {start_level, box}; + // Mesh_t mesh(box, minLevel, maxLevel); - auto level = samurai::make_field( "level", mesh ); - auto rank = samurai::make_field( "rank", mesh ); + // auto u = init( mesh, radius, x_center, y_center ); + // samurai::make_bc(u, 0.); + // auto mradapt = samurai::make_MRAdapt( u ); + // mradapt( mr_epsilon, mr_regularity ); + + std::vector clists( 3 ); + std::vector carrays( 3 ); + + clists[0][1][{0}].add_interval({1, 4}); + clists[0][1][{1}].add_interval({1, 4}); + clists[0][1][{2}].add_interval({0, 4}); + clists[0][1][{3}].add_interval({0, 4}); + clists[0][2][{0}].add_interval({0, 2}); + clists[0][2][{1}].add_interval({0, 2}); + clists[0][2][{2}].add_interval({0, 2}); + clists[0][2][{3}].add_interval({0, 2}); + + // subdomain 2 + clists[1][0][{2}].add_interval({0, 4}); + clists[1][0][{3}].add_interval({0, 4}); + + // subdomain 3 + clists[2][0][{0}].add_interval({2, 4}); + clists[2][0][{1}].add_interval({2, 4}); + + carrays[0] = { clists[0], true }; + carrays[1] = { clists[1], true }; + carrays[2] = { clists[2], true }; - samurai::for_each_cell( mesh, [&](const auto & cell ){ - level[cell] = static_cast( cell.level ); - rank [cell] = 0; - }); myTimers.stop("InitMesh"); - samurai::save( filename, mesh, level, rank ); + std::vector graph( 3 ); + { // fill graph info + + // find a way to compute this + graph[ 0 ].neighbour = {1, 2}; + graph[ 1 ].neighbour = {0, 2}; + graph[ 2 ].neighbour = {0, 1}; + + // fill "current" load ----> no MPI comm, local info + for(size_t i=0; i<3; ++i ){ + graph[ i ]._load = carrays[ i ].nb_cells(); + } + + // fill neighbour load -----> this will imply MPI comm + for(size_t i=0; i<3; ++i ){ + graph[ i ].load.resize( graph[ i ].neighbour.size() ); + for(size_t ni=0; ni( carrays, graph ); + + + CellArray_t mesh = { global, true }; + + auto init_rank = samurai::make_field( "init_rank", mesh); + + { // initialize fake_rank + + for(size_t m_=0; m_( m_ ); + }); + } + } + + } + + // output mesh + SAMURAI_LOG( "\t> Initial configurtion : " + filename ); + samurai::save( filename, mesh, init_rank ); + + myTimers.start( "load_balance_fluxes" ); + compute_load_balancing_fluxes( graph ); + myTimers.start( "load_balance_fluxes" ); + + // explicitly call load_balancing(); + + // std::cerr << "\n\n" << std::endl; + // SAMURAI_LOG( "[partition] Explicitly calling load_balancing() ... "); + + // for(int il=0; il<3; ++il ){ + // myTimers.start("load-balancing"); + // mesh.force_load_balancing(); + // myTimers.stop("load-balancing"); + + // // output mesh + // std::cerr << "\t> Dumping mesh to file : " << "load-balanced" << std::endl; + // samurai::save( "load-balanced_"+std::to_string(il), mesh, u ); + // } + + auto sfc_morton_rank = samurai::make_field( "sfc_morton_rank", mesh); + + myTimers.start( "balance_morton" ); + // perform_load_balancing_SFC( mesh, ndomains, sfc_morton_rank ); + myTimers.stop( "balance_morton" ); + + + auto diffusion_rank = samurai::make_field( "diffusion_rank", mesh); + + { // initialize fake_rank + + for(size_t m_=0; m_( m_ ); + }); + } + } + + } + + samurai::save( "load_balanced_diff_init_"+std::to_string( ndomains), mesh, sfc_morton_rank, diffusion_rank ); + + for_each_cell( mesh, [&](const auto & cell ){ + diffusion_rank[ cell ] = -1; + }); + + myTimers.start( "balance_diffusion" ); + perform_load_balancing_diffusion( mesh, carrays, ndomains, graph, diffusion_rank ); + myTimers.stop( "balance_diffusion" ); + + filename = "load_balanced_diff_" + std::to_string( ndomains); + SAMURAI_LOG( "\t> Generated file : " + filename ); + samurai::save( filename, mesh, sfc_morton_rank, diffusion_rank ); + myTimers.print(); samurai::finalize(); From d36577c154516959bb6714b5163236e659249eb2 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:40:56 +0100 Subject: [PATCH 021/170] add some unit test for load balancing function --- tests/test_loadbalancing.cpp | 89 ++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/test_loadbalancing.cpp diff --git a/tests/test_loadbalancing.cpp b/tests/test_loadbalancing.cpp new file mode 100644 index 000000000..0ead91a76 --- /dev/null +++ b/tests/test_loadbalancing.cpp @@ -0,0 +1,89 @@ +#include +#include +#include + +#include +#include +#include + +#include + +#include + +namespace samurai { + + /* + * test cmptLoad; + */ + TEST(loadBalance, cmptLoad){ + + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + + samurai::CellList cl; + cl[0][{0}].add_interval({0, 4}); + cl[0][{1}].add_interval({0, 1}); + cl[0][{1}].add_interval({3, 4}); + cl[1][{2}].add_interval({2, 6}); + cl[2][{6}].add_interval({4, 6}); + cl[2][{6}].add_interval({10, 12}); + + Mesh_t mesh( cl, 0, 2 ); + + auto ncells = cmptLoad( mesh ); + auto ninter = cmptLoad( mesh ); + + ASSERT_EQ( ncells, 14 ); + ASSERT_EQ( ninter, 6 ); + + } + + /** + * Be aware that the cmptInterface require 2:1 balance for this test + */ + TEST(loadBalance, cmptInterface){ + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + + samurai::CellList cl_a, cl_b; + + { + cl_a[0][{0}].add_interval({0, 4}); // level 0 + + cl_a[1][{2}].add_interval({0, 2}); // level 1 + cl_a[1][{3}].add_interval({0, 2}); // level 1 + cl_a[1][{2}].add_interval({6, 8}); // level 1 + cl_a[1][{3}].add_interval({6, 8}); // level 1 + + cl_a[1][{2}].add_interval({2, 6}); // level 1 + cl_a[2][{6}].add_interval({4, 6}); // level 2 + cl_a[2][{6}].add_interval({10, 12}); // level 2 + } + + { + // cl_b[0][{2}].add_interval({0, 4}); // level 0 + cl_b[0][{3}].add_interval({0, 4}); // level 0 + + cl_b[1][{4}].add_interval({0, 8}); // level 1 + cl_b[1][{5}].add_interval({0, 8}); // level 1 + + cl_b[1][{3}].add_interval({3, 5}); // level 1 + cl_b[2][{7}].add_interval({4, 6}); // level 2 + cl_b[2][{7}].add_interval({10, 12}); // level 2 + } + + Mesh_t mesh_a ( cl_a, 0, 2 ), mesh_b (cl_b, 0, 2 ); + + auto interface = cmptInterface( mesh_a, mesh_b ); + + std::cerr << interface << std::endl; + + ASSERT_EQ( interface.nb_cells(), 10 ); + + } + +} From f195e641122370d86e3daab5abd215f93b7867a8 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 22 Mar 2024 23:33:18 +0100 Subject: [PATCH 022/170] add script parse stats file --- python/stats_mpi.py | 160 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 python/stats_mpi.py diff --git a/python/stats_mpi.py b/python/stats_mpi.py new file mode 100644 index 000000000..4f3337db6 --- /dev/null +++ b/python/stats_mpi.py @@ -0,0 +1,160 @@ + +# +# +# categories = ['A', 'B', 'C', 'D'] +# valeurs_moyennes = [3, 7, 5, 9] +# valeurs_min = [2, 5, 4, 8] +# valeurs_max = [4, 9, 6, 10] + +# # Création du graphique +# plt.figure(figsize=(10, 6)) + +# # Tracé des barres +# plt.bar(categories, valeurs_moyennes, yerr=[(v_min, v_max) for v_min, v_max in zip(valeurs_min, valeurs_max)], capsize=5) + +import numpy +import re +import argparse +import pandas as pd +import matplotlib.pyplot as plt + +def plot_mma( data, suffix, ax, xlabel, ylabel, keys=None ): + """ + """ + + # select specific domaine if not None + if keys == None: + keys = data.keys() + + print("plot data for domaine {}".format(keys)) + + for mprank in keys: + levels = range( data[ mprank ].min_level.min(), data[ mprank ].max_level.max()+1 ) + + for elem in ["min", "max", "ave"]: + fields = [f'by_level.{l:02}.{suffix}.{elem}' for l in levels] + print("\t>[plot] Getting field: '{}'".format(fields)) + + # x = numpy.arange( levels.start, levels.stop ) + # y = data[ mprank ][ fields ].to_numpy()[0] + + # ax.plot( x, y, ls='None', markersize=5, marker='o' ) + + ax.set_xlabel( xlabel, fontsize=10 ) + ax.set_ylabel( ylabel, fontsize=10 ) + +def plot_by_level( data, suffix, ax, xlabel, ylabel, keys=None ): + """ + Plot data level by level + """ + + # select specific domaine if not None + if keys == None: + keys = data.keys() + + print("plot data for domaine {}".format(keys)) + + for mprank in keys: + levels = range( data[ mprank ].min_level.min(), data[ mprank ].max_level.max()+1 ) + fields = [f'by_level.{l:02}.{suffix}' for l in levels] + + x = numpy.arange( levels.start, levels.stop ) + y = data[ mprank ][ fields ].to_numpy()[0] + + ax.plot( x, y, ls='None', markersize=5, marker='o' ) + + ax.set_xlabel( xlabel, fontsize=10 ) + ax.set_ylabel( ylabel, fontsize=10 ) + + +def plot_single_quantity( data, suffix, ax, xlabel, ylabel, keys=None ): + """ + Plot data level by level + """ + + # select specific domaine if not None + if keys == None: + keys = list(data.keys()) + + rmin = int( min(keys) ) + rmax = int( max(keys) ) + + x = numpy.zeros( len(keys) ) + y = numpy.zeros( x.shape ) + + for id in range(0, len(keys)): + mprank = int( keys[ id ] ) + x[ id ] = mprank + y[ id ] = data[ str(mprank) ][suffix].values[0] + print("\t>[plot] Process # {}, data single: '{}'".format( mprank, data[ str(mprank)][suffix].values )) + + ax.plot( x, y, ls='None', markersize=5, marker='o' ) + + ax.set_xlabel( xlabel, fontsize=10 ) + ax.set_ylabel( ylabel, fontsize=10 ) + +# arguments parser +CLI = argparse.ArgumentParser() +CLI.add_argument("--files", nargs="*", type=str, default=[""], help="stats file") +CLI.add_argument("--jtag", type=str, default="stats", help="json tag") + +# parse the command line +args = CLI.parse_args() + +min_level = 99 +max_level = 0 +process_stats = {} +for fin in args.files: + + print("\t> Parsing file: '{}'".format( fin )) + fdata = pd.read_json( fin ) + fdata = pd.json_normalize( fdata[ args.jtag ] ) + + match = re.search(r'process_(\d+)', fin) + if match: + rank = match.group(1) + else: + rank = 0 + + # compute min/max level overall datasets + min_level = min( min_level, fdata.min_level.min() ) + max_level = max( max_level, fdata.max_level.max() ) + + assert rank not in process_stats.keys(), "statistics for process {} exists".format(rank) + process_stats[ rank ] = fdata + + +nrow, ncol = 3, 2 +fig = plt.figure(figsize=(5*ncol, 5*nrow)) + +# plot("cells", "Number of cells per level", "Level", "number of cells", ) + +ax = fig.add_subplot(nrow, ncol, 1) +plot_by_level( process_stats, "cells", ax, xlabel="level", ylabel="Number of cells" ) +ax.set_xlim( min_level-1, max_level+1 ) +ax.set_title( "Number of cells", fontweight="bold", fontsize=10 ) +x = numpy.arange(min_level, max_level+1) +ax.set_xticks( x, labels=x ) + +ax = fig.add_subplot(nrow, ncol, 2) +plot_by_level( process_stats, "axis-0.number of intervals", ax, xlabel="level", ylabel="Number of intervals" ) +ax.set_xlim( min_level-1, max_level+1 ) +ax.set_title( "Number of intervals", fontweight="bold", fontsize=10 ) +x = numpy.arange(min_level, max_level+1) +ax.set_xticks( x, labels=x ) + +ax = fig.add_subplot(nrow, ncol, 3) +plot_single_quantity( process_stats, "n_neighbours", ax, xlabel="MPI rank", ylabel="Number of neighbours" ) +ax.set_title( "Number of geometrical neighbours", fontweight="bold", fontsize=10 ) +# x = numpy.arange(min_level, max_level+1) +# ax.set_xticks( x, labels=x ) + +ax = fig.add_subplot(nrow, ncol, 4) +plot_mma( process_stats, "axis-0.cells per interval", ax, xlabel="level", ylabel="Number of cells" ) +ax.set_xlim( min_level-1, max_level+1 ) +ax.set_title( "Number of cells per interval x-axis", fontweight="bold", fontsize=10 ) +x = numpy.arange(min_level, max_level+1) +ax.set_xticks( x, labels=x ) + + +plt.savefig( f"{args.jtag}.png", dpi=300 ) From e08af7c4f60c9dd859be31d2dbbdc411e793cf3e Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 22 Mar 2024 23:33:38 +0100 Subject: [PATCH 023/170] add load balancing demos --- demos/loadbalancing/circle.cpp | 116 ++++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 37 deletions(-) diff --git a/demos/loadbalancing/circle.cpp b/demos/loadbalancing/circle.cpp index 946b40fb4..b51fc7f63 100644 --- a/demos/loadbalancing/circle.cpp +++ b/demos/loadbalancing/circle.cpp @@ -24,6 +24,18 @@ #include #include +template +auto getLevel(Mesh& mesh) +{ + auto level = samurai::make_field("level", mesh); + + samurai::for_each_cell( mesh, [&]( auto & cell ) { + level[ cell ] = static_cast( cell.level ); + }); + + return level; +} + template auto init(Mesh& mesh, const double radius, const double x_center, const double y_center ) { @@ -55,8 +67,13 @@ auto init(Mesh& mesh, const double radius, const double x_center, const double y template auto initMultiCircles(Mesh& mesh, const std::vector &radius, const std::vector & centers ) { + std::cerr << "\t> [InitMultiCircle] Entering ... " << std::endl; auto u = samurai::make_field("u", mesh); + samurai::for_each_cell( mesh, [&](auto& cell) { + u[ cell ] = 0.; + }); + samurai::for_each_cell( mesh, [&](auto& cell) { auto cc = cell.center(); @@ -68,9 +85,9 @@ auto initMultiCircles(Mesh& mesh, const std::vector &radius, const std:: } if ( rad <= radius[ icerc ] * radius[ icerc ]) { - u[ cell ] += 1; + u[ cell ] += 1.; } else { - u[ cell ] += 0; + u[ cell ] += 0.; } } @@ -93,9 +110,9 @@ int main( int argc, char * argv[] ){ double radius = 0.2, x_center = 0.5, y_center = 0.5; int niterBench = 1; + bool multi = false; CLI::App app{"Load balancing test"}; - app.add_option("--ndomains", ndomains, "Number of desired domains")->capture_default_str()->group("Simulation parameters"); app.add_option("--nbIterLB", nbIterLoadBalancing, "Number of desired lb iteration") ->capture_default_str()->group("Simulation parameters"); app.add_option("--niterBench", niterBench, "Number of iteration for bench")->capture_default_str()->group("Simulation parameters"); @@ -111,6 +128,7 @@ int main( int argc, char * argv[] ){ app.add_option("--radius", radius, "Bubble radius")->capture_default_str()->group("Simulation parameters"); app.add_option("--xcenter", x_center, "Bubble x-axis center")->capture_default_str()->group("Simulation parameters"); app.add_option("--ycenter", y_center, "Bubble y-axis center")->capture_default_str()->group("Simulation parameters"); + app.add_flag("--multi", multi, "Multiple bubles")->group("Simulation parameters"); CLI11_PARSE(app, argc, argv); @@ -128,9 +146,22 @@ int main( int argc, char * argv[] ){ boost::mpi::communicator world; + ndomains = static_cast( world.size() ); + + std::vector bubles_r; + std::vector> bubles_c; + + if( multi ) { + bubles_r = { 0.2, 0.1, 0.05}; + bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; + }else{ + bubles_r = { 0.25}; + bubles_c = {{0.5, 0.5}}; + } + if( world.rank() == 0 ) std::cerr << "\t> Testing Diffusion_LoadBalancer_interval " << std::endl; - { // load balancing cells by cells using gravity loadbalancer + { // load balancing cells by cells using interface propagation Timers myTimers; @@ -143,19 +174,19 @@ int main( int argc, char * argv[] ){ Mesh_t mesh(box, minLevel, maxLevel); - std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; - - auto u = init( mesh, radius, x_center, y_center ); + // auto u = init( mesh, radius, x_center, y_center ); - // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + auto u = initMultiCircles( mesh, bubles_r, bubles_c ); samurai::make_bc(u, 0.); auto mradapt = samurai::make_MRAdapt( u ); mradapt( mr_epsilon, mr_regularity ); + auto lvl = getLevel( mesh ); + myTimers.stop("InitMesh"); - samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); myTimers.start( "balance_DIF_inter" ); Diffusion_LoadBalancer_interval _diff_lb; @@ -166,11 +197,15 @@ int main( int argc, char * argv[] ){ } myTimers.stop( "balance_DIF_inter" ); - _diff_lb.evaluate_balancing( mesh ); + std::string _stats = fmt::format( "stats_diff_interval_process_{}", world.rank() ); + samurai::Statistics s ( _stats ); + s( "stats", mesh ); { Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); - auto u2 = init( newmesh, radius, x_center, y_center ); + // auto u2 = init( newmesh, radius, x_center, y_center ); + auto u2 = initMultiCircles( newmesh,bubles_r, bubles_c ); + samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); @@ -210,7 +245,7 @@ int main( int argc, char * argv[] ){ if( world.rank() == 0 ) std::cerr << "\t> Testing Diffusion_LoadBalancer_cell " << std::endl; - { // load balancing cells by cells using gravity loadbalancer + { // load balancing cells by cells using gravity Timers myTimers; @@ -223,10 +258,10 @@ int main( int argc, char * argv[] ){ Mesh_t mesh(box, minLevel, maxLevel); - std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; - - auto u = init( mesh, radius, x_center, y_center ); - // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + // auto u = init( mesh, radius, x_center, y_center ); + auto u = initMultiCircles( mesh, bubles_r, bubles_c ); + auto lvl = getLevel( mesh ); + // auto u = initMultiCircles( mesh,bubles_r, bubles_c ); samurai::make_bc(u, 0.); auto mradapt = samurai::make_MRAdapt( u ); @@ -234,7 +269,7 @@ int main( int argc, char * argv[] ){ myTimers.stop("InitMesh"); - samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + // samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); myTimers.start( "balance_DIF_cell" ); Diffusion_LoadBalancer_cell _diff_lb_c; @@ -245,11 +280,14 @@ int main( int argc, char * argv[] ){ } myTimers.stop( "balance_DIF_cell" ); - _diff_lb_c.evaluate_balancing( mesh ); + std::string _stats = fmt::format( "stats_diff_gravity_process_{}", world.rank() ); + samurai::Statistics s ( _stats ); + s( "stats", mesh ); { Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); - auto u2 = init( newmesh, radius, x_center, y_center ); + // auto u2 = init( newmesh, radius, x_center, y_center ); + auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); @@ -286,7 +324,7 @@ int main( int argc, char * argv[] ){ if( world.rank() == 0 ) std::cerr << "\t> Testing Morton_LoadBalancer_interval " << std::endl; - { // load balancing cells by cells using gravity loadbalancer + { // load balancing using SFC morton Timers myTimers; @@ -299,10 +337,10 @@ int main( int argc, char * argv[] ){ Mesh_t mesh(box, minLevel, maxLevel); - std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; - - auto u = init( mesh, radius, x_center, y_center ); - // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + // auto u = init( mesh, radius, x_center, y_center ); + auto u = initMultiCircles( mesh, bubles_r, bubles_c ); + auto lvl = getLevel( mesh ); + // auto u = initMultiCircles( mesh,bubles_r, bubles_c ); samurai::make_bc(u, 0.); auto mradapt = samurai::make_MRAdapt( u ); @@ -310,7 +348,7 @@ int main( int argc, char * argv[] ){ myTimers.stop("InitMesh"); - samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + // samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); myTimers.start( "balance_Morton_cell" ); SFC_LoadBalancer_interval loadb_morton; @@ -321,11 +359,14 @@ int main( int argc, char * argv[] ){ } myTimers.stop( "balance_Morton_cell" ); - loadb_morton.evaluate_balancing( mesh ); + std::string _stats = fmt::format( "stats_sfc_morton_process_{}", world.rank() ); + samurai::Statistics s ( _stats ); + s( "stats", mesh ); { Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); - auto u2 = init( newmesh, radius, x_center, y_center ); + // auto u2 = init( newmesh, radius, x_center, y_center ); + auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); @@ -360,7 +401,7 @@ int main( int argc, char * argv[] ){ if( world.rank() == 0 ) std::cerr << "\t> Testing Morton_LoadBalancer_interval " << std::endl; - { // load balancing cells by cells using gravity loadbalancer + { // load balancing using SFC hilbert Timers myTimers; @@ -373,18 +414,17 @@ int main( int argc, char * argv[] ){ Mesh_t mesh(box, minLevel, maxLevel); - std::vector> bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; - - auto u = init( mesh, radius, x_center, y_center ); - // auto u = initMultiCircles( mesh, {0.2, 0.1, 0.05}, bubles_c ); + // auto u = init( mesh, radius, x_center, y_center ); + auto u = initMultiCircles( mesh, bubles_r, bubles_c ); + auto lvl = getLevel( mesh ); - samurai::make_bc(u, 0.); + samurai::make_bc(u, 0.); auto mradapt = samurai::make_MRAdapt( u ); mradapt( mr_epsilon, mr_regularity ); myTimers.stop("InitMesh"); - samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh ); + // samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); myTimers.start( "balance_Hilbert_cell" ); SFC_LoadBalancer_interval loadb_hilbert; @@ -395,12 +435,14 @@ int main( int argc, char * argv[] ){ } myTimers.stop( "balance_Hilbert_cell" ); - loadb_hilbert.evaluate_balancing( mesh ); + std::string _stats = fmt::format( "stats_sfc_hilbert_process_{}", world.rank() ); + samurai::Statistics s ( _stats ); + s( "stats", mesh ); { Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); - auto u2 = init( newmesh, radius, x_center, y_center ); - + // auto u2 = init( newmesh, radius, x_center, y_center ); + auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); auto conv = samurai::make_convection(velocity); From 086b868bfa1cf4b6e9dd6d96b8deb2638e1ab988 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 2 Apr 2024 12:18:40 +0200 Subject: [PATCH 024/170] add unit test files --- tests/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6a4a5dd10..6de6d9666 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,6 +28,9 @@ set(SAMURAI_TESTS test_portion.cpp test_scaling.cpp test_utils.cpp + test_sfc.cpp + test_mrmesh.cpp + test_loadbalancing.cpp ) if(rapidcheck_FOUND) From 0ad4dab416e1dbc34f1d6a09c42f018c6dbd4abd Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 2 Apr 2024 12:19:47 +0200 Subject: [PATCH 025/170] remove cout --- include/samurai/statistics.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/samurai/statistics.hpp b/include/samurai/statistics.hpp index 5cd58327e..2e596536d 100644 --- a/include/samurai/statistics.hpp +++ b/include/samurai/statistics.hpp @@ -20,14 +20,12 @@ namespace samurai , save_all(save_all) , icurrent(0) { - std::cerr << "\t> Constucteur [Statistics] " << std::endl; } template void operator()( std::string test_case, Mesh& mesh ) { - std::cerr << "\t> Constucteur [operator()] " << std::endl; icurrent++; using mesh_id_t = typename Mesh::mesh_id_t; From 8ef2f37fea6f0608f3856dde3b7139edfb9934f1 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 2 Apr 2024 12:20:17 +0200 Subject: [PATCH 026/170] fix unused warning --- include/samurai/load_balancing_diffusion_interval.hpp | 4 ---- include/samurai/load_balancing_sfc.hpp | 1 - 2 files changed, 5 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index c20c85c8a..581ab83ff 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -82,13 +82,9 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer void load_balance_impl( Mesh_t & mesh ){ - using interval_t = typename Mesh_t::interval_t; using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; using CellList_t = typename Mesh_t::cl_type; using CellArray_t = samurai::CellArray; - using mesh_id_t = typename Mesh_t::mesh_id_t; - using Coord_t = xt::xtensor_fixed>; - using Cell_t = typename Mesh_t::cell_t; boost::mpi::communicator world; diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index b6b8b5bb4..6229b34c2 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -45,7 +45,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer; using Mesh_t = samurai::MRMesh; - using cell_t = typename Mesh_t::cell_t; using inter_t = samurai::Interval; using CellList_t = typename Mesh_t::cl_type; using CellArray_t = samurai::CellArray; From d0d76c6bca79a0e7d904d83a9bc574c2ec844bd4 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 2 Apr 2024 12:20:43 +0200 Subject: [PATCH 027/170] update load balancing demo --- demos/loadbalancing/circle.cpp | 150 +++++++++++++++++++++++++++------ 1 file changed, 124 insertions(+), 26 deletions(-) diff --git a/demos/loadbalancing/circle.cpp b/demos/loadbalancing/circle.cpp index b51fc7f63..4c6c8954b 100644 --- a/demos/loadbalancing/circle.cpp +++ b/demos/loadbalancing/circle.cpp @@ -67,7 +67,6 @@ auto init(Mesh& mesh, const double radius, const double x_center, const double y template auto initMultiCircles(Mesh& mesh, const std::vector &radius, const std::vector & centers ) { - std::cerr << "\t> [InitMultiCircle] Entering ... " << std::endl; auto u = samurai::make_field("u", mesh); samurai::for_each_cell( mesh, [&](auto& cell) { @@ -96,11 +95,26 @@ auto initMultiCircles(Mesh& mesh, const std::vector &radius, const std:: return u; } +template +void upWind(int niterBench, const Mesh_t & mesh, Field_t & u2, Field_t & unp1, double dt, Conv_t & conv) { + + for(int i=0; i( world.size() ); std::vector bubles_r; - std::vector> bubles_c; + std::vector> bubles_c; if( multi ) { bubles_r = { 0.2, 0.1, 0.05}; - bubles_c = {{0.2, 0.2}, {0.5, 0.5}, {0.8, 0.8}}; + bubles_c = {{0.2, 0.2, 0.2}, {0.5, 0.5, 0.5}, {0.8, 0.8, 0.8}}; }else{ - bubles_r = { 0.25}; - bubles_c = {{0.5, 0.5}}; + bubles_r = { 0.15}; + bubles_c = {{0.25, 0.25, 0.5}}; } if( world.rank() == 0 ) std::cerr << "\t> Testing Diffusion_LoadBalancer_interval " << std::endl; - { // load balancing cells by cells using interface propagation + if( world.size() > 1 ) { // load balancing cells by cells using interface propagation Timers myTimers; @@ -204,8 +218,7 @@ int main( int argc, char * argv[] ){ { Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); // auto u2 = init( newmesh, radius, x_center, y_center ); - auto u2 = initMultiCircles( newmesh,bubles_r, bubles_c ); - + auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); @@ -213,16 +226,23 @@ int main( int argc, char * argv[] ){ // myTimers.start( "update_ghost_mr" ); // for(int i=0; i("unp1", newmesh); myTimers.start( "upwind" ); - for(int i=0; i Testing Diffusion_LoadBalancer_cell " << std::endl; - { // load balancing cells by cells using gravity + if( world.size() > 1 ) { // load balancing cells by cells using gravity Timers myTimers; @@ -302,9 +322,10 @@ int main( int argc, char * argv[] ){ auto unp1 = samurai::make_field("unp1", newmesh); myTimers.start( "upwind" ); - for(int i=0; i Testing Morton_LoadBalancer_interval " << std::endl; - { // load balancing using SFC morton + if( world.size() > 1 ) { // load balancing using SFC morton Timers myTimers; @@ -381,9 +402,10 @@ int main( int argc, char * argv[] ){ auto unp1 = samurai::make_field("unp1", newmesh); myTimers.start( "upwind" ); - for(int i=0; i Testing Morton_LoadBalancer_interval " << std::endl; + if( world.rank() == 0 ) std::cerr << "\t> Testing Hilbert_LoadBalancer_interval " << std::endl; - { // load balancing using SFC hilbert + if( world.size() > 1 ) { // load balancing using SFC hilbert Timers myTimers; @@ -456,9 +478,10 @@ int main( int argc, char * argv[] ){ auto unp1 = samurai::make_field("unp1", newmesh); myTimers.start( "upwind" ); - for(int i=0; i Testing no loadbalancing (initial partitionning) " << std::endl; + + { // load balancing using SFC hilbert + + Timers myTimers; + + myTimers.start("global"); + + /** + * Initialize AMR mesh with spherical buble + */ + myTimers.start("InitMesh"); + + samurai::Box box( minCorner, maxCorner ); + + Mesh_t mesh(box, minLevel, maxLevel); + + // auto u = init( mesh, radius, x_center, y_center ); + auto u = initMultiCircles( mesh, bubles_r, bubles_c ); + auto lvl = getLevel( mesh ); + + samurai::make_bc(u, 0.); + auto mradapt = samurai::make_MRAdapt( u ); + mradapt( mr_epsilon, mr_regularity ); + + myTimers.stop("InitMesh"); + + std::string _stats = fmt::format( "stats_no_loadbalancing_process_{}", world.rank() ); + + myTimers.start("io"); + samurai::Statistics s ( _stats ); + s( "stats", mesh ); + myTimers.stop("io"); + + { + myTimers.start("stuff"); + Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); + // auto u2 = init( newmesh, radius, x_center, y_center ); + + + auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); + samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + + auto conv = samurai::make_convection(velocity); + + // myTimers.start( "update_ghost_mr" ); + // for(int i=0; i("unp1", newmesh); + + myTimers.stop("stuff"); + + myTimers.start( "upwind" ); + // for(int i=0; i Date: Thu, 4 Apr 2024 14:15:20 +0200 Subject: [PATCH 028/170] add void load balancing class for demo --- include/samurai/load_balancing_void.hpp | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 include/samurai/load_balancing_void.hpp diff --git a/include/samurai/load_balancing_void.hpp b/include/samurai/load_balancing_void.hpp new file mode 100644 index 000000000..7882ce2e6 --- /dev/null +++ b/include/samurai/load_balancing_void.hpp @@ -0,0 +1,36 @@ +/** + * Empty class, used for test to compare with and without load balancing. + * +*/ + +#pragma once + +#include +#include "load_balancing.hpp" + +template +class Void_LoadBalancer: public samurai::LoadBalancer> { + + private: + int _ndomains; + int _rank; + + public: + + Void_LoadBalancer() { +#ifdef SAMURAI_WITH_MPI + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); +#else + _ndomains = 1; + _rank = 0; +#endif + } + + inline std::string getName() const { return "Void_LB"; } + + template + void load_balance_impl( Mesh_t & mesh ){ } + +}; \ No newline at end of file From 2aa4f339814c5d72e4e6c8ef920dda90f4dbb93d Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 4 Apr 2024 14:18:09 +0200 Subject: [PATCH 029/170] add getName() funcitons --- include/samurai/hilbert.hpp | 4 +++- include/samurai/load_balancing_diffusion_cell.hpp | 2 ++ include/samurai/load_balancing_diffusion_interval.hpp | 2 ++ include/samurai/load_balancing_sfc.hpp | 4 +++- include/samurai/morton.hpp | 2 ++ include/samurai/sfc.hpp | 3 +++ 6 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/samurai/hilbert.hpp b/include/samurai/hilbert.hpp index 1ff9e1169..bf53476f3 100644 --- a/include/samurai/hilbert.hpp +++ b/include/samurai/hilbert.hpp @@ -70,7 +70,7 @@ class Hilbert : public SFC { for(size_t idim=0; idim 1 ){ @@ -137,6 +137,8 @@ class Hilbert : public SFC { public: + inline std::string getName() const { return "Hilbert"; } + template inline SFC_key_t getKey_2D_impl( const Coord_t & lc ) const { // SAMURAI_TRACE( "Hilbert::getKey_2D_impl Entering" ); diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 92bccb68f..e508aa3ed 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -43,6 +43,8 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer void load_balance_impl( Mesh_t & mesh ){ diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index 581ab83ff..ca9ec021b 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -79,6 +79,8 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer void load_balance_impl( Mesh_t & mesh ){ diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 6229b34c2..86bfe4771 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -40,6 +40,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer void load_balance_impl( Mesh & mesh ){ @@ -114,7 +116,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer 0 ) { diff --git a/include/samurai/morton.hpp b/include/samurai/morton.hpp index 947dc5c3d..da29d3d79 100644 --- a/include/samurai/morton.hpp +++ b/include/samurai/morton.hpp @@ -56,6 +56,8 @@ class Morton : public SFC { public: + inline std::string getName() const { return "Morton"; } + /* * Return the morton index from logical coordinate (i,j) or (i,j,k) * diff --git a/include/samurai/sfc.hpp b/include/samurai/sfc.hpp index ef67a4a45..cd2561b28 100644 --- a/include/samurai/sfc.hpp +++ b/include/samurai/sfc.hpp @@ -34,4 +34,7 @@ class SFC { if constexpr ( dim == 3 ) return static_cast( this )->getCoordinates_3D_impl( clef ); } + inline auto getName() const -> std::string { + return static_cast( this )->getName(); + } }; \ No newline at end of file From 875800f610672523724ab5ddbc6406a571df25f6 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 4 Apr 2024 14:18:31 +0200 Subject: [PATCH 030/170] clean demo code for load balancing --- demos/loadbalancing/circle.cpp | 552 +++++++++------------------------ 1 file changed, 141 insertions(+), 411 deletions(-) diff --git a/demos/loadbalancing/circle.cpp b/demos/loadbalancing/circle.cpp index 4c6c8954b..02031d77b 100644 --- a/demos/loadbalancing/circle.cpp +++ b/demos/loadbalancing/circle.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -96,12 +97,12 @@ auto initMultiCircles(Mesh& mesh, const std::vector &radius, const std:: } template -void upWind(int niterBench, const Mesh_t & mesh, Field_t & u2, Field_t & unp1, double dt, Conv_t & conv) { +void upWind(int niter_benchmark, const Mesh_t & mesh, Field_t & u2, Field_t & unp1, double dt, Conv_t & conv) { - for(int i=0; i +struct Config_test_load_balancing { + int niter_benchmark = 1; + int niter_loadbalance = 1; // number of iteration of load balancing + int ndomains = 1; // number of partition (mpi processes) + int rank = 0; // current mpi rank + std::size_t minlevel = 2; + std::size_t maxlevel = 4; + double dt = 0.1; + double mr_regularity = 1.0; + double mr_epsilon = 2.e-4; xt::xtensor_fixed> minCorner = {0., 0., 0.}; xt::xtensor_fixed> maxCorner = {1., 1., 1.}; - std::string filename = "ld_mesh_init"; + std::vector bubles_r; // bubles radius + std::vector> bubles_c; // bubles centers +}; - double radius = 0.2, x_center = 0.5, y_center = 0.5; - int niterBench = 1; - bool multi = false; - - CLI::App app{"Load balancing test"}; - app.add_option("--nbIterLB", nbIterLoadBalancing, "Number of desired lb iteration") - ->capture_default_str()->group("Simulation parameters"); - app.add_option("--niterBench", niterBench, "Number of iteration for bench")->capture_default_str()->group("Simulation parameters"); - app.add_option("--min-corner", minCorner, "The min corner of the box")->capture_default_str()->group("Simulation parameters"); - app.add_option("--max-corner", maxCorner, "The max corner of the box")->capture_default_str()->group("Simulation parameters"); - app.add_option("--min-level", minLevel, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); - app.add_option("--max-level", maxLevel, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); - app.add_option("--mr-eps", mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") - ->capture_default_str()->group("Multiresolution"); - app.add_option("--mr-reg", mr_regularity,"The regularity criteria used by the multiresolution to " - "adapt the mesh")->capture_default_str()->group("Multiresolution"); - app.add_option("--filename", filename, "File name prefix")->capture_default_str()->group("Ouput"); - app.add_option("--radius", radius, "Bubble radius")->capture_default_str()->group("Simulation parameters"); - app.add_option("--xcenter", x_center, "Bubble x-axis center")->capture_default_str()->group("Simulation parameters"); - app.add_option("--ycenter", y_center, "Bubble y-axis center")->capture_default_str()->group("Simulation parameters"); - app.add_flag("--multi", multi, "Multiple bubles")->group("Simulation parameters"); +template +Timers benchmark_loadbalancing( struct Config_test_load_balancing & conf, Load_Balancer_t && lb, Vel_t & velocity ){ - CLI11_PARSE(app, argc, argv); - - double cfl = 0.5; - double dt = cfl / (1 << maxLevel); - - // Convection operator - samurai::VelocityVector velocity; - velocity.fill(1); - velocity(1) = -1; - using Config = samurai::MRConfig; using Mesh_t = samurai::MRMesh; using mesh_id_t = typename Mesh_t::mesh_id_t; - boost::mpi::communicator world; - - ndomains = static_cast( world.size() ); - - std::vector bubles_r; - std::vector> bubles_c; - - if( multi ) { - bubles_r = { 0.2, 0.1, 0.05}; - bubles_c = {{0.2, 0.2, 0.2}, {0.5, 0.5, 0.5}, {0.8, 0.8, 0.8}}; - }else{ - bubles_r = { 0.15}; - bubles_c = {{0.25, 0.25, 0.5}}; - } + Timers myTimers; - if( world.rank() == 0 ) std::cerr << "\t> Testing Diffusion_LoadBalancer_interval " << std::endl; + /** + * Initialize AMR mesh with spherical buble + */ - if( world.size() > 1 ) { // load balancing cells by cells using interface propagation + samurai::Box box( conf.minCorner, conf.maxCorner ); - Timers myTimers; + Mesh_t mesh( box, conf.minlevel, conf.maxlevel ); - /** - * Initialize AMR mesh with spherical buble - */ - myTimers.start("InitMesh"); + // auto u = init( mesh, radius, x_center, y_center ); + myTimers.start("InitMesh"); + auto u = initMultiCircles( mesh, conf.bubles_r, conf.bubles_c ); + myTimers.stop("InitMesh"); - samurai::Box box( minCorner, maxCorner ); + auto lvl = getLevel( mesh ); - Mesh_t mesh(box, minLevel, maxLevel); + samurai::make_bc( u, 0. ); - // auto u = init( mesh, radius, x_center, y_center ); + myTimers.start("mradapt"); + auto mradapt = samurai::make_MRAdapt( u ); + mradapt( conf.mr_epsilon, conf.mr_regularity ); + myTimers.stop("mradapt"); - auto u = initMultiCircles( mesh, bubles_r, bubles_c ); - - samurai::make_bc(u, 0.); - auto mradapt = samurai::make_MRAdapt( u ); - mradapt( mr_epsilon, mr_regularity ); - - auto lvl = getLevel( mesh ); - - myTimers.stop("InitMesh"); - - samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); - - myTimers.start( "balance_DIF_inter" ); - Diffusion_LoadBalancer_interval _diff_lb; - for(size_t lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); - auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); - - samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); - - auto conv = samurai::make_convection(velocity); - - // myTimers.start( "update_ghost_mr" ); - // for(int i=0; i("unp1", newmesh); - - myTimers.start( "upwind" ); - upWind( niterBench, mesh, u2, unp1, dt, conv ); - // for(int i=0; i Testing Diffusion_LoadBalancer_cell " << std::endl; + std::string _stats = fmt::format( "stats_{}_process_{}", lb.getName(), conf.rank ); + samurai::Statistics s ( _stats ); + s( "stats", mesh ); - if( world.size() > 1 ) { // load balancing cells by cells using gravity + { + Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); + // auto u2 = init( newmesh, radius, x_center, y_center ); + auto u2 = initMultiCircles( newmesh, conf.bubles_r, conf.bubles_c ); + samurai::save( "init2_circle_"+std::to_string( conf.ndomains )+"_domains", newmesh, u2 ); - Timers myTimers; + auto conv = samurai::make_convection( velocity ); - /** - * Initialize AMR mesh with spherical buble - */ - myTimers.start("InitMesh"); + // myTimers.start( "update_ghost_mr" ); + // for(int i=0; i box( minCorner, maxCorner ); + auto unp1 = samurai::make_field("unp1", newmesh); - Mesh_t mesh(box, minLevel, maxLevel); - - // auto u = init( mesh, radius, x_center, y_center ); - auto u = initMultiCircles( mesh, bubles_r, bubles_c ); - auto lvl = getLevel( mesh ); - // auto u = initMultiCircles( mesh,bubles_r, bubles_c ); - - samurai::make_bc(u, 0.); - auto mradapt = samurai::make_MRAdapt( u ); - mradapt( mr_epsilon, mr_regularity ); - - myTimers.stop("InitMesh"); - - // samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); - - myTimers.start( "balance_DIF_cell" ); - Diffusion_LoadBalancer_cell _diff_lb_c; - for(int lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); - auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); - - samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + myTimers.start( "upwind" ); + // for(int i=0; i(velocity); - - // myTimers.start( "update_ghost_mr" ); - // for(int i=0; i("unp1", newmesh); - - myTimers.start( "upwind" ); - upWind( niterBench, mesh, u2, unp1, dt, conv ); - // for(int i=0; i Testing Morton_LoadBalancer_interval " << std::endl; - - if( world.size() > 1 ) { // load balancing using SFC morton - - Timers myTimers; - - /** - * Initialize AMR mesh with spherical buble - */ - myTimers.start("InitMesh"); - - samurai::Box box( minCorner, maxCorner ); - - Mesh_t mesh(box, minLevel, maxLevel); - - // auto u = init( mesh, radius, x_center, y_center ); - auto u = initMultiCircles( mesh, bubles_r, bubles_c ); - auto lvl = getLevel( mesh ); - // auto u = initMultiCircles( mesh,bubles_r, bubles_c ); - - samurai::make_bc(u, 0.); - auto mradapt = samurai::make_MRAdapt( u ); - mradapt( mr_epsilon, mr_regularity ); - - myTimers.stop("InitMesh"); - - // samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); - - myTimers.start( "balance_Morton_cell" ); - SFC_LoadBalancer_interval loadb_morton; - for(int lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); - auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); - - samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + return myTimers; +} - auto conv = samurai::make_convection(velocity); +int main( int argc, char * argv[] ){ - // myTimers.start( "update_ghost_mr" ); - // for(int i=0; i("unp1", newmesh); + constexpr int dim = 2; + constexpr double cfl = 0.5; - myTimers.start( "upwind" ); - // for(int i=0; i conf; - - // auto mradapt = samurai::make_MRAdapt( u2 ); - - // myTimers.start( "mradapt" ); - // mradapt( mr_epsilon, mr_regularity ); - // myTimers.stop( "mradapt" ); + std::string filename = "ld_mesh_init"; - } + double radius = 0.2, x_center = 0.5, y_center = 0.5; + bool multi = false; - myTimers.print(); + CLI::App app{"Load balancing test"}; + app.add_option("--nbIterLB", conf.niter_loadbalance, "Number of desired lb iteration") + ->capture_default_str()->group("Simulation parameters"); + app.add_option("--niterBench", conf.niter_benchmark, "Number of iteration for bench")->capture_default_str()->group("Simulation parameters"); + app.add_option("--min-level", conf.minlevel, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--max-level", conf.maxlevel, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-eps", conf.mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") + ->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-reg", conf.mr_regularity,"The regularity criteria used by the multiresolution to " + "adapt the mesh")->capture_default_str()->group("Multiresolution"); + app.add_option("--filename", filename, "File name prefix")->capture_default_str()->group("Ouput"); + app.add_option("--radius", radius, "Bubble radius")->capture_default_str()->group("Simulation parameters"); + app.add_option("--xcenter", x_center, "Bubble x-axis center")->capture_default_str()->group("Simulation parameters"); + app.add_option("--ycenter", y_center, "Bubble y-axis center")->capture_default_str()->group("Simulation parameters"); + app.add_flag("--multi", multi, "Multiple bubles")->group("Simulation parameters"); - } - - if( world.rank() == 0 ) std::cerr << "\t> Testing Hilbert_LoadBalancer_interval " << std::endl; - - if( world.size() > 1 ) { // load balancing using SFC hilbert - - Timers myTimers; - - /** - * Initialize AMR mesh with spherical buble - */ - myTimers.start("InitMesh"); - - samurai::Box box( minCorner, maxCorner ); - - Mesh_t mesh(box, minLevel, maxLevel); - - // auto u = init( mesh, radius, x_center, y_center ); - auto u = initMultiCircles( mesh, bubles_r, bubles_c ); - auto lvl = getLevel( mesh ); - - samurai::make_bc(u, 0.); - auto mradapt = samurai::make_MRAdapt( u ); - mradapt( mr_epsilon, mr_regularity ); - - myTimers.stop("InitMesh"); - - // samurai::save( "init_circle_"+std::to_string( ndomains)+"_domains", mesh, lvl ); - - myTimers.start( "balance_Hilbert_cell" ); - SFC_LoadBalancer_interval loadb_hilbert; - for(int lb_iter=0; lb_iter( newmesh, radius, x_center, y_center ); - auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); - samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); - - auto conv = samurai::make_convection(velocity); - - // myTimers.start( "update_ghost_mr" ); - // for(int i=0; i("unp1", newmesh); + CLI11_PARSE(app, argc, argv); - myTimers.start( "upwind" ); - // for(int i=0; i velocity; + velocity.fill( 1 ); + velocity( 1 ) = -1; - // myTimers.start( "mradapt" ); - // mradapt( mr_epsilon, mr_regularity ); - // myTimers.stop( "mradapt" ); + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using mesh_id_t = typename Mesh_t::mesh_id_t; - } + boost::mpi::communicator world; + conf.rank = world.rank(); + conf.ndomains = static_cast( world.size() ); - myTimers.print(); - + if( multi ) { + conf.bubles_r = { 0.2, 0.1, 0.05}; + conf.bubles_c = {{ 0.2, 0.2, 0.2 }, { 0.5, 0.5, 0.5 }, { 0.8, 0.8, 0.8 }}; + }else{ + conf.bubles_r = { radius }; + conf.bubles_c = {{ x_center, y_center, 0.5 }}; } - if( world.rank() == 0 ) std::cerr << "\t> Testing no loadbalancing (initial partitionning) " << std::endl; - - { // load balancing using SFC hilbert - - Timers myTimers; - - myTimers.start("global"); + if( conf.rank == 0 ) std::cerr << "\n\t> Testing 'Interval propagation load balancer' " << std::endl; - /** - * Initialize AMR mesh with spherical buble - */ - myTimers.start("InitMesh"); - - samurai::Box box( minCorner, maxCorner ); - - Mesh_t mesh(box, minLevel, maxLevel); - - // auto u = init( mesh, radius, x_center, y_center ); - auto u = initMultiCircles( mesh, bubles_r, bubles_c ); - auto lvl = getLevel( mesh ); - - samurai::make_bc(u, 0.); - auto mradapt = samurai::make_MRAdapt( u ); - mradapt( mr_epsilon, mr_regularity ); - - myTimers.stop("InitMesh"); - - std::string _stats = fmt::format( "stats_no_loadbalancing_process_{}", world.rank() ); - - myTimers.start("io"); - samurai::Statistics s ( _stats ); - s( "stats", mesh ); - myTimers.stop("io"); - - { - myTimers.start("stuff"); - Mesh_t newmesh( mesh[mesh_id_t::cells], mesh ); - // auto u2 = init( newmesh, radius, x_center, y_center ); - - - auto u2 = initMultiCircles( newmesh, bubles_r, bubles_c ); - samurai::save( "init2_circle_"+std::to_string( ndomains)+"_domains", newmesh, u2 ); + if( conf.ndomains > 1 ) { // load balancing cells by cells using interface propagation + auto times = benchmark_loadbalancing( conf, std::move( Diffusion_LoadBalancer_interval () ), velocity ); + times.print(); + world.barrier(); + } - auto conv = samurai::make_convection(velocity); + if( conf.rank == 0 ) std::cerr << "\n\t> Testing 'Gravity-based cell exchange' load balancer' " << std::endl; - // myTimers.start( "update_ghost_mr" ); - // for(int i=0; i 1 ) { // load balancing cells by cells using gravity + auto times = benchmark_loadbalancing( conf, std::move( Diffusion_LoadBalancer_cell () ), velocity ); + times.print(); + world.barrier(); + } - auto unp1 = samurai::make_field("unp1", newmesh); + if( conf.rank == 0 ) std::cerr << "\n\t> Testing Morton_LoadBalancer_interval " << std::endl; - myTimers.stop("stuff"); + if( conf.ndomains > 1 ) { // load balancing using SFC morton + auto times = benchmark_loadbalancing( conf, std::move( SFC_LoadBalancer_interval () ), velocity ); + times.print(); + world.barrier(); + } - myTimers.start( "upwind" ); - // for(int i=0; i Testing Hilbert_LoadBalancer_interval " << std::endl; - - // auto mradapt = samurai::make_MRAdapt( u2 ); - - // myTimers.start( "mradapt" ); - // mradapt( mr_epsilon, mr_regularity ); - // myTimers.stop( "mradapt" ); - } + if( conf.ndomains > 1 ) { // load balancing using SFC hilbert + auto times = benchmark_loadbalancing( conf, std::move( SFC_LoadBalancer_interval () ), velocity ); + times.print(); + world.barrier(); + } - myTimers.stop("global"); + if( conf.rank == 0 ) std::cerr << "\t> Testing no loadbalancing (initial partitionning) " << std::endl; - myTimers.print(); - + { + auto times = benchmark_loadbalancing( conf, std::move( Void_LoadBalancer () ), velocity ); + times.print(); + world.barrier(); } samurai::finalize(); From fab99580e89727d449a32b3fa3d1d2e9d17a2192 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 4 Apr 2024 15:35:26 +0200 Subject: [PATCH 031/170] remove useless class SFC_LoadBalancer_cells --- include/samurai/load_balancing_sfc.hpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 86bfe4771..17ce6bbca 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -4,20 +4,6 @@ #include "load_balancing.hpp" -template -class SFC_LoadBalancer_cells : public samurai::LoadBalancer> { - - private: - SFC_type_t _sfc; - - public: - template - void load_balance_impl( Mesh & mesh ){ - _sfc.getIndex( 1, 2 ); - } - -}; - template class SFC_LoadBalancer_interval : public samurai::LoadBalancer> { From 9312f9e894801462a22c30728a370200c561692d Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 15:57:10 +0200 Subject: [PATCH 032/170] fix warnings --- include/samurai/hilbert.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/samurai/hilbert.hpp b/include/samurai/hilbert.hpp index bf53476f3..7144b6d2b 100644 --- a/include/samurai/hilbert.hpp +++ b/include/samurai/hilbert.hpp @@ -147,7 +147,6 @@ class Hilbert : public SFC { assert( lc( 0 ) >= 0 ); assert( lc( 1 ) >= 0 ); - constexpr SFC_key_t one = static_cast( 1 ); constexpr int dim = 2; constexpr int _nbits = sizeof( lc( 0 ) ) * 8; @@ -185,7 +184,6 @@ class Hilbert : public SFC { assert( lc( 1 ) >= 0 ); assert( lc( 2 ) >= 0 ); - constexpr SFC_key_t one = static_cast( 1 ); constexpr int dim = 3; constexpr int _nbits = sizeof( lc( 0 ) ) * 8; From 2a8baf4a1abca58da8516f7a4229f9ac604506d8 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 15:57:31 +0200 Subject: [PATCH 033/170] remove define --- include/samurai/statistics.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/samurai/statistics.hpp b/include/samurai/statistics.hpp index 2e596536d..068d3e032 100644 --- a/include/samurai/statistics.hpp +++ b/include/samurai/statistics.hpp @@ -4,7 +4,6 @@ #include -#define WITH_STATS #if defined(WITH_STATS) #include using json = nlohmann::json; From 55fc071e9549c41c624cedd4cde853d902188b97 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 15:58:07 +0200 Subject: [PATCH 034/170] fix warning and remove old function --- demos/loadbalancing/CMakeLists.txt | 8 +- include/samurai/load_balancing.hpp | 431 ++++++++++++++--------------- 2 files changed, 220 insertions(+), 219 deletions(-) diff --git a/demos/loadbalancing/CMakeLists.txt b/demos/loadbalancing/CMakeLists.txt index b62de8e28..1d4f0b25c 100644 --- a/demos/loadbalancing/CMakeLists.txt +++ b/demos/loadbalancing/CMakeLists.txt @@ -1,5 +1,7 @@ -add_executable(test-simple-2d partition.cpp) -target_link_libraries(test-simple-2d samurai CLI11::CLI11) +include(FindPkgConfig) + +# add_executable(test-simple-2d partition.cpp) +# target_link_libraries(test-simple-2d samurai CLI11::CLI11) add_executable(test-circle-2d circle.cpp) -target_link_libraries(test-circle-2d samurai CLI11::CLI11) \ No newline at end of file +target_link_libraries(test-circle-2d samurai CLI11::CLI11 ${MPI_LIBRARIES}) \ No newline at end of file diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 660d275e7..f33416099 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -75,7 +75,7 @@ namespace samurai{ if constexpr ( elem == BalanceElement_t::CELL ) { // cell-based load without weight. samurai::for_each_interval( current_mesh, - [&]( std::size_t level, const auto& interval, const auto& index ){ + [&]( [[maybe_unused]] std::size_t level, const auto& interval, [[maybe_unused]] const auto& index ){ current_process_load += interval.size(); }); } else { @@ -363,13 +363,13 @@ namespace samurai{ samurai::for_each_cell( mesh, [&]( auto & cell ){ // all cells equals - constexpr double wght = 1.; + // constexpr double wght = 1.; // higher weight for large cells // double wght = 1. / static_cast( 1 << cell.level ); // higher weight for small cells - // double wght = 1. / static_cast( 1 << ( mesh.max_level() - cell.level ) ); + double wght = 1. / static_cast( 1 << ( mesh.max_level() - cell.level ) ); auto cc = cell.center(); @@ -506,7 +506,6 @@ namespace samurai{ for( int nbi=0; nbi -void perform_load_balancing_SFC( Mesh & mesh, int ndomains, Field_t & fake_mpi_rank ) { +// template +// void perform_load_balancing_SFC( Mesh & mesh, int ndomains, Field_t & fake_mpi_rank ) { - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; - using cell_t = typename Mesh_t::cell_t; +// using Config = samurai::MRConfig; +// using Mesh_t = samurai::MRMesh; +// using cell_t = typename Mesh::cell_t; - SFC sfc; +// SFC sfc; - // SFC key (used for implicit sorting through map mechanism) - std::map sfc_map; +// // SFC key (used for implicit sorting through map mechanism) +// std::map sfc_map; - int sfc_max_level = mesh.max_level(); // for now but remember must <= 21 for Morton +// int sfc_max_level = mesh.max_level(); // for now but remember must <= 21 for Morton - SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); +// SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); - samurai::for_each_cell( mesh, [&]( const auto & cell ){ +// samurai::for_each_cell( mesh, [&]( const auto & cell ){ - // ij coordinate of cell - double dxmax = samurai::cell_length( sfc_max_level ); +// // ij coordinate of cell +// double dxmax = samurai::cell_length( sfc_max_level ); - auto tmp = cell.center() / dxmax; +// auto tmp = cell.center() / dxmax; - xt::xtensor_fixed> ij = { static_cast( tmp( 0 ) ), - static_cast( tmp( 1 ) ) }; +// xt::xtensor_fixed> ij = { static_cast( tmp( 0 ) ), +// static_cast( tmp( 1 ) ) }; - auto key = sfc.template getKey( ij ); - // std::cerr << "\t> Coord (" << ij( 0 ) << ", " << ij( 1 ) << ") ----> " << key << std::endl; +// auto key = sfc.template getKey( ij ); +// // std::cerr << "\t> Coord (" << ij( 0 ) << ", " << ij( 1 ) << ") ----> " << key << std::endl; - sfc_map[ key ] = cell ; +// sfc_map[ key ] = cell ; - }); +// }); - size_t nbcells_tot = mesh.nb_cells( Mesh::mesh_id_t::cells ); //load -balancing on leaves - size_t ncellsPerProc = std::floor( nbcells_tot / ndomains ); +// size_t nbcells_tot = mesh.nb_cells( Mesh::mesh_id_t::cells ); //load -balancing on leaves +// size_t ncellsPerProc = std::floor( nbcells_tot / ndomains ); - // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; - std::cerr << "\t> Total number of cells : " << nbcells_tot << std::endl; - std::cout << "\t> Perfect load-balancing (weight-cell = 1.) : " << static_cast( nbcells_tot / ndomains ) - << " / MPI" << std::endl; +// // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; +// std::cerr << "\t> Total number of cells : " << nbcells_tot << std::endl; +// std::cout << "\t> Perfect load-balancing (weight-cell = 1.) : " << static_cast( nbcells_tot / ndomains ) +// << " / MPI" << std::endl; - size_t cindex = 0; - for(const auto & item : sfc_map ) { - auto sfc_key = item.first; +// size_t cindex = 0; +// for(const auto & item : sfc_map ) { +// auto sfc_key = item.first; - int fake_rank = std::floor( cindex / ncellsPerProc ); - if ( fake_rank >= ndomains ) fake_rank = ndomains - 1; +// int fake_rank = std::floor( cindex / ncellsPerProc ); +// if ( fake_rank >= ndomains ) fake_rank = ndomains - 1; - fake_mpi_rank[ item.second ] = fake_rank; - cindex ++; - } +// fake_mpi_rank[ item.second ] = fake_rank; +// cindex ++; +// } -} +// } -enum Interval_CellPosition { FIRST, LAST }; +// enum Interval_CellPosition { FIRST, LAST }; -template -void perform_load_balancing_SFC_Interval( Mesh & mesh, int ndomains, Field_t & fake_mpi_rank, const Interval_CellPosition &icp ) { +// template +// void perform_load_balancing_SFC_Interval( Mesh & mesh, int ndomains, Field_t & fake_mpi_rank, const Interval_CellPosition &icp ) { - using inter_t = samurai::Interval; +// using inter_t = samurai::Interval; - struct Data { - size_t level; - inter_t interval; - xt::xtensor_fixed> indices; - }; +// struct Data { +// size_t level; +// inter_t interval; +// xt::xtensor_fixed> indices; +// }; - // SFC key (used for implicit sorting through map mechanism) - std::map sfc_map; +// // SFC key (used for implicit sorting through map mechanism) +// std::map sfc_map; - SFC sfc; - int sfc_max_level = mesh.max_level(); // for now but remember must <= 21 for Morton +// SFC sfc; +// int sfc_max_level = mesh.max_level(); // for now but remember must <= 21 for Morton - SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); +// SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); - size_t ninterval = 0; - samurai::for_each_interval( mesh, [&]( std::size_t level, const auto& inter, const auto& index ){ +// size_t ninterval = 0; +// samurai::for_each_interval( mesh, [&]( std::size_t level, const auto& inter, const auto& index ){ - // get Logical coordinate or first cell - xt::xtensor_fixed> icell; +// // get Logical coordinate or first cell +// xt::xtensor_fixed> icell; - icp == Interval_CellPosition::LAST ? icell( 0 ) = inter.end - 1 : icell( 0 ) = inter.start; +// icp == Interval_CellPosition::LAST ? icell( 0 ) = inter.end - 1 : icell( 0 ) = inter.start; - for(int idim=0; idim> ijk; +// xt::xtensor_fixed> ijk; - if constexpr ( dim == 2 ) ijk = { static_cast( icell( 0 ) ), - static_cast( icell( 1 ) ) }; +// if constexpr ( dim == 2 ) ijk = { static_cast( icell( 0 ) ), +// static_cast( icell( 1 ) ) }; - if constexpr ( dim == 3 ) ijk = { static_cast( icell( 0 ) ), - static_cast( icell( 1 ) ), - static_cast( icell( 2 ) ) }; +// if constexpr ( dim == 3 ) ijk = { static_cast( icell( 0 ) ), +// static_cast( icell( 1 ) ), +// static_cast( icell( 2 ) ) }; - sfc_map[ sfc.getKey( ijk ) ] = { level, inter, index }; +// sfc_map[ sfc.getKey( ijk ) ] = { level, inter, index }; - ninterval ++; - }); +// ninterval ++; +// }); - size_t ninterPerProc = std::floor( ninterval / ndomains ); +// size_t ninterPerProc = std::floor( ninterval / ndomains ); - // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; - std::cerr << "\t> Total number of interval : " << ninterval << std::endl; - std::cout << "\t> Perfect load-balancing (weight-interval = 1.) : " << static_cast( ninterPerProc ) - << " / MPI" << std::endl; +// // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; +// std::cerr << "\t> Total number of interval : " << ninterval << std::endl; +// std::cout << "\t> Perfect load-balancing (weight-interval = 1.) : " << static_cast( ninterPerProc ) +// << " / MPI" << std::endl; - size_t cindex = 0; - for(const auto & item : sfc_map ) { - int fake_rank = std::floor( cindex / ninterPerProc ); - if ( fake_rank >= ndomains ) fake_rank = ndomains - 1; +// size_t cindex = 0; +// for(const auto & item : sfc_map ) { +// int fake_rank = std::floor( cindex / ninterPerProc ); +// if ( fake_rank >= ndomains ) fake_rank = ndomains - 1; - fake_mpi_rank( item.second.level, item.second.interval, item.second.indices ) = fake_rank; +// fake_mpi_rank( item.second.level, item.second.interval, item.second.indices ) = fake_rank; - cindex ++; - } +// cindex ++; +// } -} +// } -/** - * - * Global contains the global mesh. Since this is a toy function, it contains the union of all mesh. - * meshes contains the mesh of each MPI process ( or let say the mesh of neighbour processes ...) - * -*/ -template -void perform_load_balancing_diffusion( AMesh_t & global, std::vector & meshes, int ndomains, - const std::vector & all, Field_t & fake_mpi_rank ) { +// /** +// * +// * Global contains the global mesh. Since this is a toy function, it contains the union of all mesh. +// * meshes contains the mesh of each MPI process ( or let say the mesh of neighbour processes ...) +// * +// */ +// template +// void perform_load_balancing_diffusion( AMesh_t & global, std::vector & meshes, int ndomains, +// const std::vector & all, Field_t & fake_mpi_rank ) { - using CellList_t = typename AMesh_t::cl_type; +// using CellList_t = typename AMesh_t::cl_type; - struct Coord_t { xt::xtensor_fixed> coord; }; +// struct Coord_t { xt::xtensor_fixed> coord; }; - std::vector barycenters ( ndomains ); +// std::vector barycenters ( ndomains ); - { // compute barycenter of current domains ( here, all domains since with simulate multiple MPI domains) +// { // compute barycenter of current domains ( here, all domains since with simulate multiple MPI domains) - int maxlevel = 4; - for( size_t m_=0; m_ < meshes.size(); ++m_ ) { +// int maxlevel = 4; +// for( size_t m_=0; m_ < meshes.size(); ++m_ ) { - double wght_tot = 0.; - samurai::for_each_cell( meshes[ m_ ], [&]( const auto & cell ) { +// double wght_tot = 0.; +// samurai::for_each_cell( meshes[ m_ ], [&]( const auto & cell ) { - // [OPTIMIZATION] precompute weight as array - double wght = 1. / ( 1 << ( maxlevel - cell.level ) ); +// // [OPTIMIZATION] precompute weight as array +// double wght = 1. / ( 1 << ( maxlevel - cell.level ) ); - const auto cc = cell.center(); +// const auto cc = cell.center(); - barycenters[ m_ ].coord( 0 ) += cc( 0 ) * wght; - barycenters[ m_ ].coord( 1 ) += cc( 1 ) * wght; - if constexpr ( dim == 3 ) { barycenters[ m_ ].coord( 2 ) += cc( 2 ) * wght; } +// barycenters[ m_ ].coord( 0 ) += cc( 0 ) * wght; +// barycenters[ m_ ].coord( 1 ) += cc( 1 ) * wght; +// if constexpr ( dim == 3 ) { barycenters[ m_ ].coord( 2 ) += cc( 2 ) * wght; } - wght_tot += wght; +// wght_tot += wght; - }); +// }); - barycenters[ m_ ].coord( 0 ) /= wght_tot; - barycenters[ m_ ].coord( 1 ) /= wght_tot; - if constexpr ( dim == 3 ) barycenters[ m_ ].coord( 2 ) /= wght_tot; +// barycenters[ m_ ].coord( 0 ) /= wght_tot; +// barycenters[ m_ ].coord( 1 ) /= wght_tot; +// if constexpr ( dim == 3 ) barycenters[ m_ ].coord( 2 ) /= wght_tot; - std::cerr << "\t> Domain # " << m_ << ", bc : {" << barycenters[ m_ ].coord( 0 ) << ", " - << barycenters[ m_ ].coord( 1 ) << "}" << std::endl; +// std::cerr << "\t> Domain # " << m_ << ", bc : {" << barycenters[ m_ ].coord( 0 ) << ", " +// << barycenters[ m_ ].coord( 1 ) << "}" << std::endl; - } +// } - } +// } - std::vector new_meshes( ndomains ); - std::vector exchanged( ndomains ); +// std::vector new_meshes( ndomains ); +// std::vector exchanged( ndomains ); - for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains - std::cerr << "\t> Working on domains # " << m_ << std::endl; +// std::cerr << "\t> Working on domains # " << m_ << std::endl; - // auto dist = samurai::make_field( "rank", meshes[ m_ ] ); +// // auto dist = samurai::make_field( "rank", meshes[ m_ ] ); - // id des neighbours dans le tableau de la structure MPI_Load_Balance - // attention différent du rank mpi ! - std::vector id_send; +// // id des neighbours dans le tableau de la structure MPI_Load_Balance +// // attention différent du rank mpi ! +// std::vector id_send; - int n_neighbours = static_cast( all[ m_ ].neighbour.size() ); - for(std::size_t nbi = 0; nbi( all[ m_ ].neighbour.size() ); +// for(std::size_t nbi = 0; nbi Neighbour rank : " << nbi_rank << std::endl; - std::cerr << "\t\t> Neighbour flux : " << all[ m_ ].fluxes[ nbi ] << std::endl; +// std::cerr << "\t\t> Neighbour rank : " << nbi_rank << std::endl; +// std::cerr << "\t\t> Neighbour flux : " << all[ m_ ].fluxes[ nbi ] << std::endl; - if( nbCellsToTransfer < 0 ){ +// if( nbCellsToTransfer < 0 ){ - id_send.emplace_back( nbi ); +// id_send.emplace_back( nbi ); - // Logical_coord_t stencil; - // { // Compute the stencil or normalized direction to neighbour - // Coord_t tmp; - // double n2 = 0; - // for(int idim = 0; idim( tmp.coord( idim ) / 0.5 ); - // } - - // std::cerr << "\t\t> stencil for neighbour # " << nbi_rank << " : "; - // for(size_t idim=0; idim( tmp.coord( idim ) / 0.5 ); +// // } + +// // std::cerr << "\t\t> stencil for neighbour # " << nbi_rank << " : "; +// // for(size_t idim=0; idim Number RECV : " << id_send.size() << std::endl; +// std::cerr << "\t\t> Number RECV : " << id_send.size() << std::endl; - std::vector already_given( n_neighbours, 0 ); +// std::vector already_given( n_neighbours, 0 ); - for_each_interval( meshes[ m_ ], [&]( std::size_t level, const auto& interval, const auto& index ){ - Coord_t ibar; +// for_each_interval( meshes[ m_ ], [&]( std::size_t level, const auto& interval, const auto& index ){ +// Coord_t ibar; - std::cerr << "\t\t> Interval [" << interval.start << ", " << interval.end << "[" << std::endl; +// std::cerr << "\t\t> Interval [" << interval.start << ", " << interval.end << "[" << std::endl; - double dm = 1. / (1 << level ); - ibar.coord( 0 ) = ( (interval.end - interval.start) * 0.5 + interval.start ) * dm ; - ibar.coord( 1 ) = ( index( 1 ) * dm ) + dm * 0.5; - if constexpr ( dim == 3 ) ibar( 2 ) = ( index( 2 ) * dm ) + dm * 0.5; +// double dm = 1. / (1 << level ); +// ibar.coord( 0 ) = ( (interval.end - interval.start) * 0.5 + interval.start ) * dm ; +// ibar.coord( 1 ) = ( index( 1 ) * dm ) + dm * 0.5; +// if constexpr ( dim == 3 ) ibar( 2 ) = ( index( 2 ) * dm ) + dm * 0.5; - std::cerr << "\t\t\t> level " << level << " center @ {" << ibar.coord(0) << ", " << ibar.coord(1) << "}" << std::endl; +// std::cerr << "\t\t\t> level " << level << " center @ {" << ibar.coord(0) << ", " << ibar.coord(1) << "}" << std::endl; - // find which neighbour will potentially receive this interval - int winner_id = -1; // keep it to current if still negative - double winner_dist = std::numeric_limits::max(); - for( int nbi=0; nbi::max(); +// for( int nbi=0; nbi Dist to neighbour #" << neighbour_rank << " : " << dist << " vs " << winner_dist << std::endl; - std::cerr << "\t\t\t> Already given to this neighbour " << already_given[ neighbour_rank ] << std::endl; - std::cerr << "\t\t\t> NbCells of this interval " << interval.size() << std::endl; - std::cerr << "\t\t\t> fluxes for this neighbour : " << all[ m_ ].fluxes[ id_send[ nbi ] ] << std::endl; +// std::cerr << "\t\t\t> Dist to neighbour #" << neighbour_rank << " : " << dist << " vs " << winner_dist << std::endl; +// std::cerr << "\t\t\t> Already given to this neighbour " << already_given[ neighbour_rank ] << std::endl; +// std::cerr << "\t\t\t> NbCells of this interval " << interval.size() << std::endl; +// std::cerr << "\t\t\t> fluxes for this neighbour : " << all[ m_ ].fluxes[ id_send[ nbi ] ] << std::endl; - if( dist < winner_dist && - already_given[ neighbour_rank ] + interval.size() <= (- all[ m_ ].fluxes[ id_send[ nbi ] ]) ){ +// if( dist < winner_dist && +// already_given[ neighbour_rank ] + interval.size() <= (- all[ m_ ].fluxes[ id_send[ nbi ] ]) ){ - winner_id = id_send[ nbi ]; - winner_dist = dist; - } +// winner_id = id_send[ nbi ]; +// winner_dist = dist; +// } - } +// } - if( winner_id >= 0 ){ - auto neighbour_rank = all[ m_ ].neighbour[ winner_id ]; - std::cerr << "\t> Interval given to process " << neighbour_rank << " + ncells : " << interval.size() << std::endl; - exchanged[ neighbour_rank ][ level ][ index ].add_interval( interval ); - already_given[ neighbour_rank ] += interval.size(); - }else{ - new_meshes[ m_ ][ level ][ index ].add_interval( interval ); - } +// if( winner_id >= 0 ){ +// auto neighbour_rank = all[ m_ ].neighbour[ winner_id ]; +// std::cerr << "\t> Interval given to process " << neighbour_rank << " + ncells : " << interval.size() << std::endl; +// exchanged[ neighbour_rank ][ level ][ index ].add_interval( interval ); +// already_given[ neighbour_rank ] += interval.size(); +// }else{ +// new_meshes[ m_ ][ level ][ index ].add_interval( interval ); +// } - }); +// }); - meshes[ m_ ] = { new_meshes[ m_ ], true }; +// meshes[ m_ ] = { new_meshes[ m_ ], true }; - } +// } - for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains - for( int level=global.min_level(); level<=global.max_level(); ++level ) { - auto intersect = intersection( global[ level ], meshes[ m_ ][ level ]); +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains +// for( int level=global.min_level(); level<=global.max_level(); ++level ) { +// auto intersect = intersection( global[ level ], meshes[ m_ ][ level ]); - intersect([&]( auto & i, auto & index ) { - fake_mpi_rank( level, i, index ) = static_cast( m_ ); - }); - } +// intersect([&]( auto & i, auto & index ) { +// fake_mpi_rank( level, i, index ) = static_cast( m_ ); +// }); +// } - AMesh_t tmp = { exchanged[ m_], true }; - for( int level=global.min_level(); level<=global.max_level(); ++level ) { - auto intersect = intersection( global[ level ], tmp[ level ]); +// AMesh_t tmp = { exchanged[ m_], true }; +// for( int level=global.min_level(); level<=global.max_level(); ++level ) { +// auto intersect = intersection( global[ level ], tmp[ level ]); - intersect([&]( auto & i, auto & index ) { - fake_mpi_rank( level, i, index ) = static_cast( m_ ); - }); - } - } +// intersect([&]( auto & i, auto & index ) { +// fake_mpi_rank( level, i, index ) = static_cast( m_ ); +// }); +// } +// } -} +// } // template // void perform_load_balancing_diffusion( AMesh_t & global, int ndomains, const std::vector & all, Field_t & fake_mpi_rank ) { @@ -1293,4 +1292,4 @@ void perform_load_balancing_diffusion( AMesh_t & global, std::vector & // } // } -// } \ No newline at end of file +// } From 020b530827a866b13ed5efa38076e8bf87115308 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 15:58:53 +0200 Subject: [PATCH 035/170] fix warnings --- demos/loadbalancing/circle.cpp | 29 +++++++++---------- .../samurai/load_balancing_diffusion_cell.hpp | 3 -- .../load_balancing_diffusion_interval.hpp | 4 +-- include/samurai/load_balancing_void.hpp | 2 +- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/demos/loadbalancing/circle.cpp b/demos/loadbalancing/circle.cpp index 02031d77b..750e04014 100644 --- a/demos/loadbalancing/circle.cpp +++ b/demos/loadbalancing/circle.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -101,7 +102,7 @@ void upWind(int niter_benchmark, const Mesh_t & mesh, Field_t & u2, Field_t & un for(int i=0; i & conf, L auto conv = samurai::make_convection( velocity ); - // myTimers.start( "update_ghost_mr" ); - // for(int i=0; i("unp1", newmesh); @@ -247,10 +248,6 @@ int main( int argc, char * argv[] ){ velocity.fill( 1 ); velocity( 1 ) = -1; - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; - using mesh_id_t = typename Mesh_t::mesh_id_t; - boost::mpi::communicator world; conf.rank = world.rank(); conf.ndomains = static_cast( world.size() ); @@ -266,7 +263,7 @@ int main( int argc, char * argv[] ){ if( conf.rank == 0 ) std::cerr << "\n\t> Testing 'Interval propagation load balancer' " << std::endl; if( conf.ndomains > 1 ) { // load balancing cells by cells using interface propagation - auto times = benchmark_loadbalancing( conf, std::move( Diffusion_LoadBalancer_interval () ), velocity ); + auto times = benchmark_loadbalancing( conf, Diffusion_LoadBalancer_interval (), velocity ); times.print(); world.barrier(); } @@ -274,7 +271,7 @@ int main( int argc, char * argv[] ){ if( conf.rank == 0 ) std::cerr << "\n\t> Testing 'Gravity-based cell exchange' load balancer' " << std::endl; if( conf.ndomains > 1 ) { // load balancing cells by cells using gravity - auto times = benchmark_loadbalancing( conf, std::move( Diffusion_LoadBalancer_cell () ), velocity ); + auto times = benchmark_loadbalancing( conf, Diffusion_LoadBalancer_cell (), velocity ); times.print(); world.barrier(); } @@ -282,7 +279,7 @@ int main( int argc, char * argv[] ){ if( conf.rank == 0 ) std::cerr << "\n\t> Testing Morton_LoadBalancer_interval " << std::endl; if( conf.ndomains > 1 ) { // load balancing using SFC morton - auto times = benchmark_loadbalancing( conf, std::move( SFC_LoadBalancer_interval () ), velocity ); + auto times = benchmark_loadbalancing( conf, SFC_LoadBalancer_interval (), velocity ); times.print(); world.barrier(); } @@ -290,7 +287,7 @@ int main( int argc, char * argv[] ){ if( conf.rank == 0 ) std::cerr << "\n\t> Testing Hilbert_LoadBalancer_interval " << std::endl; if( conf.ndomains > 1 ) { // load balancing using SFC hilbert - auto times = benchmark_loadbalancing( conf, std::move( SFC_LoadBalancer_interval () ), velocity ); + auto times = benchmark_loadbalancing( conf, SFC_LoadBalancer_interval (), velocity ); times.print(); world.barrier(); } @@ -298,12 +295,12 @@ int main( int argc, char * argv[] ){ if( conf.rank == 0 ) std::cerr << "\t> Testing no loadbalancing (initial partitionning) " << std::endl; { - auto times = benchmark_loadbalancing( conf, std::move( Void_LoadBalancer () ), velocity ); + auto times = benchmark_loadbalancing( conf, Void_LoadBalancer (), velocity ); times.print(); world.barrier(); } samurai::finalize(); - return EXIT_SUCCESS; + return 0; } \ No newline at end of file diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index e508aa3ed..c02bf30ed 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -187,9 +187,6 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancersecond.rank ].rank; - auto lvl = it->second.cell.level; - // shouldn't we give it to the second closest neighbour ?! if( given_[ it->second.rank ] + 1 <= ( - fluxes[ it->second.rank ] ) ){ diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index ca9ec021b..f77e9edce 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -153,7 +153,8 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer; using mesh_id_t = typename Mesh_t::mesh_id_t; - using Coord_t = xt::xtensor_fixed>; boost::mpi::communicator world; diff --git a/include/samurai/load_balancing_void.hpp b/include/samurai/load_balancing_void.hpp index 7882ce2e6..0def67d05 100644 --- a/include/samurai/load_balancing_void.hpp +++ b/include/samurai/load_balancing_void.hpp @@ -31,6 +31,6 @@ class Void_LoadBalancer: public samurai::LoadBalancer> { inline std::string getName() const { return "Void_LB"; } template - void load_balance_impl( Mesh_t & mesh ){ } + void load_balance_impl( [[maybe_unused]] Mesh_t & mesh ){ } }; \ No newline at end of file From b31976122ef5c76c22a571e32c17225100ff98b7 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 15:59:58 +0200 Subject: [PATCH 036/170] fix warnings --- include/samurai/mesh.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index ea086bca2..785808527 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -830,7 +830,7 @@ namespace samurai // remove cells cl_type cl; size_t diff_ncells = 0; - for( int ilvl=refmesh.min_level(); ilvl<=refmesh.max_level(); ++ilvl ) { + for( size_t ilvl=refmesh.min_level(); ilvl<=refmesh.max_level(); ++ilvl ) { auto diff = samurai::difference( refmesh[ ilvl ], lca[ ilvl ] ); From ee3b1d037c6016102520e083c8e1eb19344b8267 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 16:00:19 +0200 Subject: [PATCH 037/170] fxi read stats scripts --- python/read_stats.py | 101 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 93 insertions(+), 8 deletions(-) diff --git a/python/read_stats.py b/python/read_stats.py index adb4f9193..1f13c2337 100644 --- a/python/read_stats.py +++ b/python/read_stats.py @@ -1,15 +1,85 @@ + +# +# +# categories = ['A', 'B', 'C', 'D'] +# valeurs_moyennes = [3, 7, 5, 9] +# valeurs_min = [2, 5, 4, 8] +# valeurs_max = [4, 9, 6, 10] + +# # Création du graphique +# plt.figure(figsize=(10, 6)) + +# # Tracé des barres +# plt.bar(categories, valeurs_moyennes, yerr=[(v_min, v_max) for v_min, v_max in zip(valeurs_min, valeurs_max)], capsize=5) + +import numpy +import argparse import pandas as pd import matplotlib.pyplot as plt -test_case = "D2Q4444_Euler_Lax_Liu" +# arguments parser +CLI = argparse.ArgumentParser() + +# list of PDF files +CLI.add_argument("--file", type=str, default=[""], help="stats file") -data = pd.read_json('stats.json') +# parse the command line +args = CLI.parse_args() + +test_case = "stats" + +data = pd.read_json( args.file ) data = pd.json_normalize(data[test_case]) +print(type(print(data.min_level))) +print(data.min_level) + min_level = data.min_level.min() max_level = data.max_level.max() levels = range(min_level, max_level+1) +print("> Levels: {}".format(levels)) + +def plot_minmax( suffix, title, xlabel, ylabel, ax, level_range, kind='box', legend=None): + """ + level_range (in) : range of levels (min_level, max_level) + """ + print("\t> [plot_minmax] Entering .. ") + # keys in dict. of data + _tmp_fields = [] + _tmp_name = [] + _minmaxave = {} + for elem in ["min", "max", "ave"]: + _fields = [f'by_level.{l:02}.{suffix}.{elem}' for l in level_range] + + _minmaxave[elem] = data[ _fields ].to_numpy()[0] + + _tmp_fields.append( _fields ) + _tmp_name.append( {f: l for f, l in zip(_fields, levels)} ) + + # ax.set_xlabel(xlabel) + # ax.set_ylabel(ylabel) + ax.set_xlim( level_range.start-1, level_range.stop ) + ax.set_title(title, fontweight="bold", fontsize=10) + + x = numpy.arange(min_level, max_level+1) + spacing = 0.3 # spacing between hat groups + width = (1 - spacing) / x.shape[0] + ax.set_xticks( x, labels=x ) + + ax.errorbar( x, _minmaxave["ave"], yerr=[_minmaxave["ave"]-_minmaxave["min"], _minmaxave["max"]-_minmaxave["ave"]], \ + ls='None', marker='o', ms=6, capsize=5, c="b" ) + + # heights0 = values[0] + # for i, (heights, group_label) in enumerate(zip(values, group_labels)): + # style = {'fill': False} if i == 0 else {'edgecolor': 'black'} + # rects = ax.bar(x - spacing/2 + i * width, heights - heights0, + # width, bottom=heights0, label=group_label, **style) + # label_bars(heights, rects) + + # print(to_plot) + # print(type(to_plot)) + def plot(suffix, title, xlabel, ylabel, ax, kind='box', legend=None, stacked=True): fields = [f'by_level.{l:02}.{suffix}' for l in levels] new_name = {f: l for f, l in zip(fields, levels)} @@ -18,20 +88,35 @@ def plot(suffix, title, xlabel, ylabel, ax, kind='box', legend=None, stacked=Tru to_plot.plot(kind=kind, ax=ax, stacked=stacked) ax.set_xlabel(xlabel) ax.set_ylabel(ylabel) - ax.set_title(title, fontweight="bold") + ax.set_title(title, fontweight="bold", fontsize=10) if legend: ax.legend(title=legend) nrow, ncol = 3, 4 fig = plt.figure(figsize=(5*ncol, 5*nrow)) -plot("cells", "Number of cells per level", "Time iteration", "number of cells", fig.add_subplot(nrow, 2, 1), kind='area', legend="Level") +plot("cells", "Number of cells per level", "Time iteration", "number of cells", fig.add_subplot(nrow, 2, 1), kind="box", legend="Level") + +plot("axis-0.number of intervals", "Number of intervals per level in x-axis", "Time iteration", \ + "number of intervals", fig.add_subplot(nrow, 2, 3), kind='box', legend="Level", stacked=False) + +plot("axis-1.number of intervals", "Number of intervals per level in y-axis", "Time iteration", \ + "number of intervals", fig.add_subplot(nrow, 2, 5), kind='box', legend="Level", stacked=False) + +# plot min/max numbers of cells per level +# plot("axis-0.cells per interval.min", "Minimum of cells per interval\n in x-axis", "level", \ +# "number of cells", fig.add_subplot(nrow, ncol, 3)) + +# plot("axis-0.cells per interval.max", "Maximum of cells per interval\n in x-axis", "level", \ +# "number of cells", fig.add_subplot(nrow, ncol, 4)) -plot("axis-0.number of intervals", "Number of intervals per level in x-axis", "Time iteration", "number of intervals", fig.add_subplot(nrow, 2, 3), kind='area', legend="Level", stacked=False) -plot("axis-1.number of intervals", "Number of intervals per level in y-axis", "Time iteration", "number of intervals", fig.add_subplot(nrow, 2, 5), kind='area', legend="Level", stacked=False) +ax = fig.add_subplot(nrow, ncol, 3) +plot_minmax("axis-0.cells per interval", "[Min, Max, Ave] Cells per interval in x-axis", "Level", \ + "number of cells", fig.add_subplot(nrow, ncol, 3), level_range=levels, kind='box', legend="Level") +plot_minmax("axis-1.cells per interval", "[Min, Max, Ave] Cells per interval in y-axis", "Level", \ + "number of cells", fig.add_subplot(nrow, ncol, 4), level_range=levels, kind='box', legend="Level") +# ax.bar(numpy.arange(min_level, max_level+1), valeurs_moyennes, yerr=[(v_min, v_max) for v_min, v_max in zip(valeurs_min, valeurs_max)], capsize=5) -plot("axis-0.cells per interval.min", "Minimum of cells per interval\n in x-axis", "level", "number of cells", fig.add_subplot(nrow, ncol, 3)) -plot("axis-0.cells per interval.max", "Maximum of cells per interval\n in x-axis", "level", "number of cells", fig.add_subplot(nrow, ncol, 4)) plot("axis-1.cells per interval.min", "Minimum of cells per interval\n in y-axis", "level", "number of cells", fig.add_subplot(nrow, ncol, 7)) plot("axis-1.cells per interval.max", "Maximum of cells per interval\n in y-axis", "level", "number of cells", fig.add_subplot(nrow, ncol, 8)) From d80d6adec52549fd6ce4ff9c235738517b56759d Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 16:00:44 +0200 Subject: [PATCH 038/170] fix load balancing sfc transfer rate --- include/samurai/load_balancing_sfc.hpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 17ce6bbca..bd5578abf 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -12,6 +12,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer sfc_map; - // FIXME: should a parameter of the function to adjust level ? - int sfc_max_level = currentMesh.max_level(); - - // boundaries of keys in current process - SFC_key_t min=std::numeric_limits::max(), max=std::numeric_limits::min(); - size_t ninterval = 0; samurai::for_each_interval( mesh, [&]( std::size_t level, const auto& inter, const auto& index ){ @@ -107,14 +101,14 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer 0 ) { neighbour_rank_prev = _rank - 1; - // transfer 50 % max of difference - transfer_load_prev = - ( my_load_i - load_interval[ neighbour_rank_prev ] ) * 0.5; + // transfer TRANSFER_PERCENT % max of difference + transfer_load_prev = - static_cast( ( my_load_i - load_interval[ neighbour_rank_prev ] ) * TRANSFER_PERCENT ); } if( _rank < _ndomains - 1 ) { neighbour_rank_next = _rank + 1; - // transfer 50 % max of difference - transfer_load_next = - ( my_load_i - load_interval[ neighbour_rank_next ] ) * 0.5; + // transfer TRANSFER_PERCENT % max of difference + transfer_load_next = - static_cast( ( my_load_i - load_interval[ neighbour_rank_next ] ) * TRANSFER_PERCENT ); } logs << "Neighbour prev : " << neighbour_rank_prev << ", loads : " << transfer_load_prev << std::endl; From b9d80eb322423e2478de914b5de7052ea67ea38a Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 16:06:15 +0200 Subject: [PATCH 039/170] fix warning --- include/samurai/load_balancing.hpp | 2 +- include/samurai/load_balancing_diffusion_cell.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index f33416099..9f4d8e69d 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -695,7 +695,7 @@ namespace samurai{ int maxlevel_check = std::min( static_cast( meshA_.max_level() ), static_cast( level ) + 1 ); - for(int projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + for(size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ // translate meshB_ in "dir" direction auto set = translate( meshB_[ level ], stencil ); diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index c02bf30ed..86c7bf0ed 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -138,7 +138,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer::max(); // double winner_dist = samurai::getDistance( cell, barycenter ) / loads[ world.rank() ]; - double coeff_current = currentSV / loads[ world.rank() ]; + double coeff_current = currentSV / loads[ static_cast( world.rank() ) ]; double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; // select the neighbour From 81364205cdd124402e4fd35f9acd50e52c16e876 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 12 Apr 2024 13:52:09 +0200 Subject: [PATCH 040/170] fix warnings --- demos/loadbalancing/circle.cpp | 38 +++-- include/samurai/hilbert.hpp | 144 +++++++++--------- .../samurai/load_balancing_diffusion_cell.hpp | 34 ++--- .../load_balancing_diffusion_interval.hpp | 30 ++-- include/samurai/load_balancing_sfc.hpp | 14 +- 5 files changed, 134 insertions(+), 126 deletions(-) diff --git a/demos/loadbalancing/circle.cpp b/demos/loadbalancing/circle.cpp index 750e04014..c57ca2363 100644 --- a/demos/loadbalancing/circle.cpp +++ b/demos/loadbalancing/circle.cpp @@ -26,6 +26,8 @@ #include #include +#include + template auto getLevel(Mesh& mesh) { @@ -118,8 +120,8 @@ struct Config_test_load_balancing { int niter_loadbalance = 1; // number of iteration of load balancing int ndomains = 1; // number of partition (mpi processes) int rank = 0; // current mpi rank - std::size_t minlevel = 2; - std::size_t maxlevel = 4; + std::size_t minlevel = 6; + std::size_t maxlevel = 8; double dt = 0.1; double mr_regularity = 1.0; double mr_epsilon = 2.e-4; @@ -181,27 +183,31 @@ Timers benchmark_loadbalancing( struct Config_test_load_balancing & conf, L samurai::save( "init2_circle_"+std::to_string( conf.ndomains )+"_domains", newmesh, u2 ); auto conv = samurai::make_convection( velocity ); - - myTimers.start( "update_ghost_mr" ); - for(int i=0; i("unp1", newmesh); + // myTimers.start( "update_ghost_mr_bf" ); + // for(int i=0; icapture_default_str()->group("Simulation parameters"); - app.add_option("--niterBench", conf.niter_benchmark, "Number of iteration for bench")->capture_default_str()->group("Simulation parameters"); + app.add_option("--iter-bench", conf.niter_benchmark, "Number of iteration for bench")->capture_default_str()->group("Simulation parameters"); app.add_option("--min-level", conf.minlevel, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); app.add_option("--max-level", conf.maxlevel, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); app.add_option("--mr-eps", conf.mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") diff --git a/include/samurai/hilbert.hpp b/include/samurai/hilbert.hpp index 7144b6d2b..0ca52adca 100644 --- a/include/samurai/hilbert.hpp +++ b/include/samurai/hilbert.hpp @@ -8,44 +8,46 @@ class Hilbert : public SFC { private: // const int _nbits = sizeof(int) * 8; - const std::vector>> _2D_STATE = {{{1, 0, 2, 3}, {0, 0, 2, 2}}, - {{0, 3, 2, 1}, {1, 3, 1, 3}}, - {{2, 1, 0, 3}, {3, 1, 3, 1}}, - {{0, 1, 3, 2}, {2, 2, 0, 0}}}; - - const std::vector>> _3D_STATE = {{{1,2,0,6,11,4,5,6,10,4,7,10}, - {0,0,0,2,4,6,4,6,2,2,4,6}}, - {{2,6,9,0,11,4,7,1,3,4,2,3}, - {1,7,3,3,3,5,7,7,5,1,5,1}}, - {{3,0,10,6,0,8,5,6,1,8,11,2}, - {3,1,7,1,5,1,3,5,3,5,7,7}}, - {{2,7,9,11,7,8,3,10,1,8,2,6}, - {2,6,4,0,2,2,0,4,4,6,6,0}}, - {{4,8,1,9,5,0,1,9,10,2,7,10}, - {7,3,1,5,7,7,5,1,1,3,3,5}}, - {{5,8,1,0,9,6,1,4,3,7,5,3}, - {6,4,2,4,0,4,6,0,6,0,2,2}}, - {{3,0,11,9,0,10,11,9,5,2,8,4}, - {4,2,6,6,6,0,2,2,0,4,0,4}}, - {{5,7,11,8,7,6,11,10,9,3,5,4}, - {5,5,5,7,1,3,1,3,7,7,1,3}}}; - - const std::vector>> _3D_STATE_r = {{{1,2,0,11,9,10,3,4,5,7,8,6}, - {0,0,0,3,5,6,3,5,6,5,6,3}}, - {{2,0,1,6,7,8,11,9,10,4,5,3}, - {1,2,4,2,7,2,7,4,4,1,7,1}}, - {{2,0,1,6,7,8,11,9,10,4,5,3}, - {3,6,5,0,3,3,6,6,0,0,5,5}}, - {{3,8,9,0,11,6,5,10,1,2,7,4}, - {2,4,1,1,1,7,2,7,2,4,4,7}}, - {{3,8,9,0,11,6,5,10,1,2,7,4}, - {6,5,3,5,0,5,0,3,3,6,0,6}}, - {{5,7,11,9,0,4,1,6,3,8,2,10}, - {7,7,7,4,2,1,4,2,1,2,1,4}}, - {{5,7,11,9,0,4,1,6,3,8,2,10}, - {5,3,6,6,6,0,5,0,5,3,3,0}}, - {{4,6,10,8,5,0,7,1,9,3,11,2}, - {4,1,2,7,4,4,1,1,7,7,2,2}}}; + // State based algorithm + + // const std::vector>> _2D_STATE = {{{1, 0, 2, 3}, {0, 0, 2, 2}}, + // {{0, 3, 2, 1}, {1, 3, 1, 3}}, + // {{2, 1, 0, 3}, {3, 1, 3, 1}}, + // {{0, 1, 3, 2}, {2, 2, 0, 0}}}; + + // const std::vector>> _3D_STATE = {{{1,2,0,6,11,4,5,6,10,4,7,10}, + // {0,0,0,2,4,6,4,6,2,2,4,6}}, + // {{2,6,9,0,11,4,7,1,3,4,2,3}, + // {1,7,3,3,3,5,7,7,5,1,5,1}}, + // {{3,0,10,6,0,8,5,6,1,8,11,2}, + // {3,1,7,1,5,1,3,5,3,5,7,7}}, + // {{2,7,9,11,7,8,3,10,1,8,2,6}, + // {2,6,4,0,2,2,0,4,4,6,6,0}}, + // {{4,8,1,9,5,0,1,9,10,2,7,10}, + // {7,3,1,5,7,7,5,1,1,3,3,5}}, + // {{5,8,1,0,9,6,1,4,3,7,5,3}, + // {6,4,2,4,0,4,6,0,6,0,2,2}}, + // {{3,0,11,9,0,10,11,9,5,2,8,4}, + // {4,2,6,6,6,0,2,2,0,4,0,4}}, + // {{5,7,11,8,7,6,11,10,9,3,5,4}, + // {5,5,5,7,1,3,1,3,7,7,1,3}}}; + + // const std::vector>> _3D_STATE_r = {{{1,2,0,11,9,10,3,4,5,7,8,6}, + // {0,0,0,3,5,6,3,5,6,5,6,3}}, + // {{2,0,1,6,7,8,11,9,10,4,5,3}, + // {1,2,4,2,7,2,7,4,4,1,7,1}}, + // {{2,0,1,6,7,8,11,9,10,4,5,3}, + // {3,6,5,0,3,3,6,6,0,0,5,5}}, + // {{3,8,9,0,11,6,5,10,1,2,7,4}, + // {2,4,1,1,1,7,2,7,2,4,4,7}}, + // {{3,8,9,0,11,6,5,10,1,2,7,4}, + // {6,5,3,5,0,5,0,3,3,6,0,6}}, + // {{5,7,11,9,0,4,1,6,3,8,2,10}, + // {7,7,7,4,2,1,4,2,1,2,1,4}}, + // {{5,7,11,9,0,4,1,6,3,8,2,10}, + // {5,3,6,6,6,0,5,0,5,3,3,0}}, + // {{4,6,10,8,5,0,7,1,9,3,11,2}, + // {4,1,2,7,4,4,1,1,7,7,2,2}}}; // def __init__(self, indexbits=16, dim=2): @@ -284,49 +286,49 @@ class Hilbert : public SFC { // return la_clef; // } - template - inline SFC_key_t getKey_3D_impl( const Coord_t & lc, int lvl ) const { - SFC_key_t la_clef = 0; + // template + // inline SFC_key_t getKey_3D_impl( const Coord_t & lc, int lvl ) const { + // SFC_key_t la_clef = 0; - constexpr int dim = 3; + // constexpr int dim = 3; - int twotondim = 1 << dim; - int ind_bits[lvl][dim]; - int h_digits[lvl]; + // int twotondim = 1 << dim; + // int ind_bits[lvl][dim]; + // int h_digits[lvl]; - SFC_key_t xyz[ dim ] = { lc( 0 ), lc( 1 ), lc( 2 ) }; + // SFC_key_t xyz[ dim ] = { lc( 0 ), lc( 1 ), lc( 2 ) }; - // Set ind_bits for current point - for (int ibit = 0; ibit < lvl; ++ibit) { - for(size_t idim=0; idim> ibit ) & 1; - } - } + // // Set ind_bits for current point + // for (int ibit = 0; ibit < lvl; ++ibit) { + // for(size_t idim=0; idim> ibit ) & 1; + // } + // } - // Compute Hilbert key bits - int cur_state = 0; - int new_state; - for(int ibit=lvl-1; ibit>-1; --ibit) { + // // Compute Hilbert key bits + // int cur_state = 0; + // int new_state; + // for(int ibit=lvl-1; ibit>-1; --ibit) { - // Compute s_digit by interleaving bits - int s_digit = 0; - for(size_t idim=0; idim( std::pow(twotondim, ibit) * h_digits[ibit] ); + // } - return la_clef; - } + // return la_clef; + // } }; diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 86c7bf0ed..63073281b 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -53,7 +53,6 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer; using Cell_t = typename Mesh_t::cell_t; using mesh_id_t = typename Mesh_t::mesh_id_t; - using Coord_t = xt::xtensor_fixed>; boost::mpi::communicator world; @@ -65,10 +64,10 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer & neighbourhood = mesh.mpi_neighbourhood(); - size_t n_neighbours = neighbourhood.size(); + std::size_t n_neighbours = neighbourhood.size(); std::vector loads; - double my_load = static_cast( cmptLoad( mesh ) ); + double my_load = static_cast( samurai::cmptLoad( mesh ) ); boost::mpi::all_gather( world, my_load, loads ); // get the load to neighbours (geometrical neighbour) @@ -90,11 +89,11 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( mesh ); + auto interface = samurai::_computeCartesianInterface( mesh ); // compute some point of reference in mesh and interval-based interface // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); - Coord_t barycenter = _cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = samurai::_cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; // std::vector barycenter_interface_neighbours( n_neighbours ); @@ -104,7 +103,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( interface[ nbi ] ); - barycenter_neighbours[ nbi ] = _cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); + barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); sv[ nbi ] = getSurfaceOrVolume( neighbourhood[ nbi ].mesh ); @@ -142,9 +141,9 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( cc, barycenter ) * coeff_current; // select the neighbour - for( size_t ni=0; ni( neighbourhood[ ni ].rank ); if( fluxes[ ni ] >= 0 ) continue; // skip neighbour that will recv @@ -159,7 +158,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( ni ); winner_dist = dist; } @@ -187,21 +186,23 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( it->second.rank ); + // shouldn't we give it to the second closest neighbour ?! - if( given_[ it->second.rank ] + 1 <= ( - fluxes[ it->second.rank ] ) ){ + if( given_[ rank ] + 1 <= ( - fluxes[ rank ] ) ){ if constexpr ( dim == 3 ) { auto i = it->second.cell.indices[ 0 ]; auto j = it->second.cell.indices[ 1 ]; auto k = it->second.cell.indices[ 2 ]; - cl_to_send[ it->second.rank ][ it->second.cell.level ][ { j, k } ].add_point( i ); + cl_to_send[ rank ][ it->second.cell.level ][ { j, k } ].add_point( i ); }else{ auto i = it->second.cell.indices[ 0 ]; auto j = it->second.cell.indices[ 1 ]; - cl_to_send[ it->second.rank ][ it->second.cell.level ][ { j } ].add_point( i ); + cl_to_send[ rank ][ it->second.cell.level ][ { j } ].add_point( i ); } - given_[ it->second.rank ] += 1; + given_[ rank ] += 1; } @@ -249,12 +250,9 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( mesh ); - requireNextIter = samurai::discover_neighbour( mesh ); - // } + samurai::discover_neighbour( mesh ); + samurai::discover_neighbour( mesh ); } diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index f77e9edce..b4fc041f5 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -101,7 +101,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer loads; - int my_load = static_cast( cmptLoad( mesh ) ); + int my_load = static_cast( samurai::cmptLoad( mesh ) ); boost::mpi::all_gather( world, my_load, loads ); std::vector & neighbourhood = mesh.mpi_neighbourhood(); @@ -125,17 +125,19 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer= 0 ) continue; if( - new_fluxes[ nbi ] > requested_load ){ - requested_load = - new_fluxes[ nbi ]; - requester = static_cast( nbi ); + requested_load = - new_fluxes[ nbi ]; + requester = nbi; + neighbour_found = true; } } - if( requester < 0 ){ + if( ! neighbour_found ){ { // debug logs << "No more neighbour found " << std::endl; } @@ -149,7 +151,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer( mesh, neighbourhood[ requester ].mesh ); + auto interface = samurai::cmptInterface( mesh, neighbourhood[ requester ].mesh ); { // check emptyness of interface, if it is empty, then set fluxes for this neighbour to 0 size_t nelement = 0; @@ -225,8 +227,8 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer( mesh ); - requireNextIter = samurai::discover_neighbour( mesh ); + samurai::discover_neighbour( mesh ); + samurai::discover_neighbour( mesh ); } @@ -257,7 +259,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer fluxes = samurai::cmptFluxes( mesh ); { - logs << "load : " << cmptLoad( mesh ) << std::endl; + logs << "load : " << samurai::cmptLoad( mesh ) << std::endl; logs << "nneighbours : " << n_neighbours << std::endl; logs << "neighbours : "; for( size_t in=0; in( mesh ); + auto interface = samurai::_computeCartesianInterface( mesh ); // compute some point of reference in mesh and interval-based interface // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); - Coord_t barycenter = _cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = samurai::_cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; std::vector invloads; - double my_load = static_cast( cmptLoad( mesh ) ); + double my_load = static_cast( samurai::cmptLoad( mesh ) ); boost::mpi::all_gather( world, my_load, invloads ); for(size_t il=0; il( interface[ nbi ] ); - barycenter_neighbours[ nbi ] = _cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); + barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); // debug // auto s_ = fmt::format( "Barycenter neighbour : ({}, {})", diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index bd5578abf..bdd68b366 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -26,7 +26,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer load_interval; - int my_load_i = static_cast( cmptLoad( mesh ) ); + int my_load_i = static_cast( samurai::cmptLoad( mesh ) ); boost::mpi::all_gather( world, my_load_i, load_interval ); // compute load to transfer to neighbour rank-1, rank+1 @@ -102,13 +102,13 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer 0 ) { neighbour_rank_prev = _rank - 1; // transfer TRANSFER_PERCENT % max of difference - transfer_load_prev = - static_cast( ( my_load_i - load_interval[ neighbour_rank_prev ] ) * TRANSFER_PERCENT ); + transfer_load_prev = - static_cast( ( my_load_i - load_interval[ static_cast( neighbour_rank_prev ) ] ) * TRANSFER_PERCENT ); } if( _rank < _ndomains - 1 ) { neighbour_rank_next = _rank + 1; // transfer TRANSFER_PERCENT % max of difference - transfer_load_next = - static_cast( ( my_load_i - load_interval[ neighbour_rank_next ] ) * TRANSFER_PERCENT ); + transfer_load_next = - static_cast( ( my_load_i - load_interval[ static_cast( neighbour_rank_next ) ] ) * TRANSFER_PERCENT ); } logs << "Neighbour prev : " << neighbour_rank_prev << ", loads : " << transfer_load_prev << std::endl; @@ -186,8 +186,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( mesh ); - requireNextIter = samurai::discover_neighbour( mesh ); + samurai::discover_neighbour( mesh ); + samurai::discover_neighbour( mesh ); } From f82cce5e9503e62eb103f93dbfe52bc3349c3938 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 12 Apr 2024 13:52:56 +0200 Subject: [PATCH 041/170] fix warnings --- include/samurai/load_balancing.hpp | 99 +++++++++++++----------------- include/samurai/statistics.hpp | 6 +- 2 files changed, 46 insertions(+), 59 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 9f4d8e69d..9a26710ac 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -17,10 +17,10 @@ namespace samurai{ struct MPI_Load_Balance { - size_t _load; + int32_t _load; std::vector neighbour; - std::vector load; - std::vector fluxes; + std::vector load; + std::vector fluxes; }; enum Distance_t { L1, L2, LINF, GRAVITY }; @@ -64,13 +64,13 @@ namespace samurai{ * mesh_id_t::cells to only consider leaves. */ template - static size_t cmptLoad( const Mesh_t & mesh ) { + static std::size_t cmptLoad( const Mesh_t & mesh ) { using mesh_id_t = typename Mesh_t::mesh_id_t; const auto & current_mesh = mesh[ mesh_id_t::cells ]; - size_t current_process_load = 0; + std::size_t current_process_load = 0; if constexpr ( elem == BalanceElement_t::CELL ) { // cell-based load without weight. @@ -80,7 +80,7 @@ namespace samurai{ }); } else { // interval-based load without weight - for( size_t level=current_mesh.min_level(); level<=current_mesh.max_level(); ++level ){ + for( std::size_t level=current_mesh.min_level(); level<=current_mesh.max_level(); ++level ){ current_process_load += current_mesh[ level ].shape()[ 0 ]; // only in x-axis ; } } @@ -129,11 +129,11 @@ namespace samurai{ int my_load_new = my_load; for( std::size_t n_i = 0; n_i < n_neighbours; ++n_i ){ - int neighbour_rank = neighbourhood[ n_i ].rank; + std::size_t neighbour_rank = static_cast( neighbourhood[ n_i ].rank ); int neighbour_load = loads[ neighbour_rank ]; double diff_load = static_cast( neighbour_load - my_load ); - size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[ neighbour_rank ]; + std::size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[ neighbour_rank ]; double weight = 1. / static_cast( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); @@ -201,8 +201,8 @@ namespace samurai{ static_cast(this)->load_balance_impl( mesh, kw... ); } - template - void evaluate_balancing( Mesh & mesh, Args&... kw ) const { + template + void evaluate_balancing( Mesh & mesh ) const { boost::mpi::communicator world; @@ -255,7 +255,8 @@ namespace samurai{ using base_stencil = xt::xtensor_fixed>; xt::xtensor_fixed> stencils; - for( int ist=0; ist<2*dim; ++ist ){ + std::size_t nstencils = static_cast( 2 * dim ); + for( std::size_t ist=0; ist + template static auto cmptInterface( Mesh_t & mesh, Mesh_t & omesh ){ using CellList_t = typename Mesh_t::cl_type; @@ -415,18 +416,15 @@ namespace samurai{ const auto & stencil = dirs[ ist ]; - // size_t nbCellsIntersection = 0; - // size_t nbIntervalIntersection = 0; size_t minlevel = otherMesh.min_level(); size_t maxlevel = otherMesh.max_level(); for( size_t level=minlevel; level<=maxlevel; ++level ) { // for each level we need to check level -1 / 0 / +1 - int minlevel_check = std::max( static_cast( currentMesh.min_level() ), - static_cast( level ) - leveldiff ); - int maxlevel_check = std::min( static_cast( currentMesh.max_level() ), - static_cast( level ) + leveldiff ); + std::size_t minlevel_check = static_cast( std::max( static_cast( currentMesh.min_level() ), + static_cast( level ) - static_cast( leveldiff ) ) ); + std::size_t maxlevel_check = std::min( currentMesh.max_level(), level + leveldiff ); for(size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ @@ -435,7 +433,7 @@ namespace samurai{ auto intersect = intersection( set, currentMesh[ projlevel ] ).on( projlevel ); size_t nbInter_ = 0, nbCells = 0; - intersect([&]( auto & interval, auto & index ) { + intersect([&]( auto & interval, [[maybe_unused]] auto & index ) { nbInter_ += 1; nbCells += interval.size(); }); @@ -494,7 +492,7 @@ namespace samurai{ // give access to geometricaly neighbour process rank and mesh std::vector & neighbourhood = mesh.mpi_neighbourhood(); - int n_neighbours = static_cast( neighbourhood.size() ); + std::size_t n_neighbours = neighbourhood.size(); std::vector interface( n_neighbours ); @@ -504,7 +502,7 @@ namespace samurai{ // looks like using only FACE is better than using DIAG or FACE_AND_DIAG auto dirs = getDirection(); - for( int nbi=0; nbi( currentMesh.min_level() ), - static_cast( level ) - 1 ); - int maxlevel_check = std::min( static_cast( currentMesh.max_level() ), - static_cast( level ) + 1 ); + std::size_t minlevel_check = static_cast( std::max( static_cast( currentMesh.min_level() ), static_cast( level ) - 1 ) ); + std::size_t maxlevel_check = std::min( currentMesh.max_level() , level + static_cast( 1 ) ); - for(size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + for(std::size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ // translate neighbour from dir (hopefully to current) all direction are tested auto set = translate( neighbour_mesh[ level ], stencil ); auto intersect = intersection( set, currentMesh[ projlevel ] ).on( projlevel ); size_t nbInter_ = 0, nbCells_ = 0; - intersect([&]( auto & interval, auto & index ) { + intersect([&]( auto & interval, [[maybe_unused]] auto & index ) { nbInter_ += 1; nbCells_ += interval.size(); }); @@ -558,9 +553,6 @@ namespace samurai{ } } - nbCellsIntersection = nbCells_ ; - nbIntervalIntersection = nbInter_ ; - } } @@ -570,7 +562,7 @@ namespace samurai{ } std::vector interface_( n_neighbours ); - for(size_t nbi=0; nbi( all[ irank ].neighbour[ j_rank ] ); auto neighbour_load = all[ irank ].load[ j_rank ]; - int64_t diff_load = neighbour_load - load; + auto diff_load = neighbour_load - load; - int nb_neighbours_neighbour = all[ neighbour_rank ].neighbour.size(); + std::size_t nb_neighbours_neighbour = all[ neighbour_rank ].neighbour.size(); - double weight = 1. / ( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); + double weight = 1. / static_cast( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); - int64_t transfertLoad = static_cast( std::lround( weight * diff_load ) ); + int32_t transfertLoad = static_cast( std::lround( weight * static_cast( diff_load ) ) ); all[ irank].fluxes[ j_rank ] += transfertLoad; @@ -654,7 +646,7 @@ namespace samurai{ // load_transfer( load_fluxes ); - load = static_cast(load_np1); + load = load_np1; } } @@ -683,33 +675,29 @@ namespace samurai{ const auto & stencil = dirs[ ist ]; - size_t nbIntervalIntersection = 0; size_t minlevel = meshB_.min_level(); size_t maxlevel = meshB_.max_level(); for( size_t level=minlevel; level<=maxlevel; ++level ) { // for each level we need to check level -1 / 0 / +1 - int minlevel_check = std::max( static_cast( meshA_.min_level() ), - static_cast( level ) - 1 ); - int maxlevel_check = std::min( static_cast( meshA_.max_level() ), - static_cast( level ) + 1 ); + std::size_t minlevel_check = static_cast( std::max( static_cast( meshA_.min_level() ), + static_cast( level ) - 1 ) ); + std::size_t maxlevel_check = std::min( meshA_.max_level(), level + static_cast( 1 ) ); - for(size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + for(std::size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ // translate meshB_ in "dir" direction auto set = translate( meshB_[ level ], stencil ); auto intersect = intersection( set, meshA_[ projlevel ] ).on( projlevel ); size_t nbInter_ = 0; - intersect([&]( auto & interval, auto & index ) { + intersect([&]( [[maybe_unused]] auto & interval, [[maybe_unused]] auto & index ) { nbInter_ += 1; }); if( nbInter_ > 0 ) return true; - nbIntervalIntersection += nbInter_ ; - } } @@ -724,7 +712,7 @@ namespace samurai{ * Discover new neighbour connection that might arise during load balancing */ template - bool discover_neighbour( Mesh_t & mesh ) { + void discover_neighbour( Mesh_t & mesh ) { using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; @@ -737,9 +725,8 @@ namespace samurai{ // give access to geometricaly neighbour process rank and mesh std::vector & neighbourhood = mesh.mpi_neighbourhood(); - int n_neighbours = static_cast( neighbourhood.size() ); - bool requireGeneralUpdate = false; + // bool requireGeneralUpdate = false; std::vector keepNeighbour( neighbourhood.size(), true ); @@ -752,7 +739,7 @@ namespace samurai{ keepNeighbour[ nbi ] = intersectionExists( mesh, neighbourhood[ nbi ].mesh ); // require update if a connection is lost - requireGeneralUpdate = requireGeneralUpdate || ( ! keepNeighbour[ nbi ] ); + // requireGeneralUpdate = requireGeneralUpdate || ( ! keepNeighbour[ nbi ] ); if( ! keepNeighbour[ nbi ] ){ logs << fmt::format("Loosing neighbour connection with {}", neighbourhood[ nbi ].rank) << std::endl; @@ -802,7 +789,7 @@ namespace samurai{ logs << "\t\t> New neighbour detected : " << _rcv_neighbour_list[ in ] << std::endl; _tmp[ _rcv_neighbour_list[ in ] ] = true; - requireGeneralUpdate = requireGeneralUpdate || true; + // requireGeneralUpdate = requireGeneralUpdate || true; } } @@ -827,7 +814,7 @@ namespace samurai{ // gather neighbour mesh mesh.update_mesh_neighbour(); - return requireGeneralUpdate; + // return requireGeneralUpdate; } } // namespace samurai diff --git a/include/samurai/statistics.hpp b/include/samurai/statistics.hpp index 068d3e032..4e58aa676 100644 --- a/include/samurai/statistics.hpp +++ b/include/samurai/statistics.hpp @@ -14,10 +14,10 @@ namespace samurai #if defined(WITH_STATS) struct Statistics { - Statistics( const std::string & filename, int save_all = 10) + Statistics( const std::string & filename, std::size_t save_all = 10) : _outfile( filename ) - , save_all(save_all) , icurrent(0) + , save_all(save_all) { } @@ -115,7 +115,7 @@ namespace samurai std::string _outfile; json stats; std::size_t icurrent; - int save_all; + std::size_t save_all; }; #else From fbf90ae5ce219ce46ef8cb505ee595978cb1d9cf Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 17 Apr 2024 11:08:50 +0200 Subject: [PATCH 042/170] fix partition mesh with 1 mpi --- include/samurai/mesh.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 785808527..dfe18c242 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -297,7 +297,7 @@ namespace samurai construct_subdomain(); m_domain = m_subdomain; construct_union(); - update_sub_mesh(); + update_sub_mesh(); // MPI AllReduce inside renumbering(); update_mesh_neighbour(); @@ -689,7 +689,8 @@ namespace samurai int product_of_sizes = 1; for (std::size_t d = 0; d < dim - 1; ++d) { - sizes[d] = static_cast(floor(pow(size, 1. / dim) * global_box.length()[d] / length_harmonic_avg)); + sizes[d] = std::max( static_cast(floor(pow(size, 1. / dim) * global_box.length()[d] / length_harmonic_avg)), 1 ); + product_of_sizes *= sizes[d]; } sizes[dim - 1] = size / product_of_sizes; @@ -843,7 +844,6 @@ namespace samurai // new mesh for current process refmesh = { cl, false }; - } template From a49af140fc100210812d85b37b1815eea74f77ef Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 17 Apr 2024 11:09:08 +0200 Subject: [PATCH 043/170] fix libdeps error in cmake --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1dc957071..3dc1117b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,7 +239,7 @@ endif() # Package the project package_project( - TARGETS samurai project_options project_warnings libdeps + TARGETS samurai project_options project_warnings INTERFACE_DEPENDENCIES_CONFIGURED ${DEPENDENCIES_CONFIGURED} INTERFACE_INCLUDES ${INCLUDE_DIR} ) From 76b20478766be1038ff19da5dcc9a11acce215e2 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 17 Apr 2024 11:09:40 +0200 Subject: [PATCH 044/170] add load balancing in advection 2D demo + fix new interface bc --- demos/FiniteVolume/advection_2d.cpp | 33 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 10d2d90f2..3b24e1c1f 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -15,6 +15,9 @@ #include #include +#include +#include + #include #include @@ -157,8 +160,7 @@ void save(const fs::path& path, const std::string& filename, const Field& u, con fs::create_directory(path); } - boost::mpi::communicator world; - int mrank = world.rank(); + int mrank = 0; samurai::for_each_cell(mesh, [&](const auto& cell) @@ -168,6 +170,7 @@ void save(const fs::path& path, const std::string& filename, const Field& u, con }); #ifdef SAMURAI_WITH_MPI mpi::communicator world; + mrank = world.rank(); samurai::save(path, fmt::format("{}_size_{}{}", filename, world.size(), suffix), mesh, u, level_); #else samurai::save(path, fmt::format("{}{}", filename, suffix), mesh, u, level_); @@ -186,11 +189,11 @@ int main(int argc, char* argv[]) // Simulation parameters double radius = 0.2, x_center = 0.3, y_center = 0.3; xt::xtensor_fixed> min_corner = {0., 0.}; - xt::xtensor_fixed> max_corner = {4., 7.}; + xt::xtensor_fixed> max_corner = {4., 4.}; std::array a{ {1, 1} }; - double Tf = .1; + double Tf = .9; double cfl = 0.5; // Multiresolution parameters @@ -236,7 +239,7 @@ int main(int argc, char* argv[]) const double dt_save = Tf / static_cast(nfiles); double t = 0.; - auto u = init(mesh); + auto u = init( mesh, radius, x_center, y_center ); samurai::make_bc>(u, 0.); auto unp1 = samurai::make_field("unp1", mesh); @@ -250,8 +253,21 @@ int main(int argc, char* argv[]) std::size_t nsave = 1; std::size_t nt = 0; + SFC_LoadBalancer_interval balancer; + while (t != Tf) { + + if( nt % 20 == 0 && nt > 1 ){ + std::cout << "\t> Load balancing mesh ... " << std::endl; + myTimers.start("load-balancing"); + balancer.load_balance( mesh ); + myTimers.stop("load-balancing"); + + unp1.resize(); + u.resize(); + } + myTimers.start( "MRadaptation" ); MRadaptation(mr_epsilon, mr_regularity); myTimers.stop( "MRadaptation" ); @@ -283,9 +299,10 @@ int main(int argc, char* argv[]) std::swap(u.array(), unp1.array()); myTimers.start( "I/O" ); - if (t >= static_cast(nsave + 1) * dt_save || t == Tf) - { - const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; + // if (t >= static_cast(nsave + 1) * dt_save || t == Tf || true ) + if( nt % 20 == 0 ) { + // const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; + const std::string suffix = fmt::format("_ite_{}", nt); save(path, filename, u, suffix); } myTimers.stop( "I/O" ); From 81c217034a74f74535fde14f20ade043103e565c Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 17 Apr 2024 11:09:59 +0200 Subject: [PATCH 045/170] fix bc and conv interface --- demos/loadbalancing/circle.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demos/loadbalancing/circle.cpp b/demos/loadbalancing/circle.cpp index c57ca2363..e27b9c8b6 100644 --- a/demos/loadbalancing/circle.cpp +++ b/demos/loadbalancing/circle.cpp @@ -155,7 +155,7 @@ Timers benchmark_loadbalancing( struct Config_test_load_balancing & conf, L auto lvl = getLevel( mesh ); - samurai::make_bc( u, 0. ); + samurai::make_bc>( u, 0. ); myTimers.start("mradapt"); auto mradapt = samurai::make_MRAdapt( u ); @@ -182,7 +182,7 @@ Timers benchmark_loadbalancing( struct Config_test_load_balancing & conf, L auto u2 = initMultiCircles( newmesh, conf.bubles_r, conf.bubles_c ); samurai::save( "init2_circle_"+std::to_string( conf.ndomains )+"_domains", newmesh, u2 ); - auto conv = samurai::make_convection( velocity ); + auto conv = samurai::make_convection_upwind( velocity ); auto unp1 = samurai::make_field("unp1", newmesh); // myTimers.start( "update_ghost_mr_bf" ); @@ -298,7 +298,7 @@ int main( int argc, char * argv[] ){ world.barrier(); } - if( conf.rank == 0 ) std::cerr << "\t> Testing no loadbalancing (initial partitionning) " << std::endl; + if( conf.rank == 0 ) std::cerr << "\n\t> Testing no loadbalancing (initial partitionning) " << std::endl; { auto times = benchmark_loadbalancing( conf, Void_LoadBalancer (), velocity ); From df98cb28a18706fddc55e1eda1fceebeebd73de8 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 17 Apr 2024 11:12:34 +0200 Subject: [PATCH 046/170] extract merge / remove CA from Mesh from mesh file to load balance file --- include/samurai/load_balancing.hpp | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 9a26710ac..6f83c80ef 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -15,6 +15,60 @@ #include namespace samurai{ + + namespace load_balance { + + template + Mesh_t merge( Mesh_t & mesh, const CellArray_t & lca ) { + using cl_type = typename Mesh_t::cl_type; + + auto & refmesh = mesh[ Mesh_t::mesh_id_t::cells ]; + + auto minlevel = std::min( refmesh.min_level(), lca.min_level() ); + auto maxlevel = std::max( refmesh.max_level(), lca.max_level() ); + + cl_type cl; + for( size_t ilvl=minlevel; ilvl<=maxlevel; ++ilvl ) { + + auto un = samurai::union_( refmesh[ ilvl ], lca[ ilvl ] ); + + un([&]( auto & interval, auto & indices ) { + cl[ ilvl ][ indices ].add_interval( interval ); + }); + } + + return Mesh_t( cl, minlevel, maxlevel ); + + } + + template + Mesh_t remove( Mesh_t & mesh, CellArray_t & lca ) { + using cl_type = typename Mesh_t::cl_type; + + auto & refmesh = mesh[ Mesh_t::mesh_id_t::cells ]; + + auto minlevel = std::min( refmesh.min_level(), lca.min_level() ); + auto maxlevel = std::max( refmesh.max_level(), lca.max_level() ); + + // remove cells + cl_type cl; + size_t diff_ncells = 0; + for( size_t ilvl=minlevel; ilvl<=maxlevel; ++ilvl ) { + + auto diff = samurai::difference( refmesh[ ilvl ], lca[ ilvl ] ); + + diff([&]( auto & interval, auto & index ) { + cl[ ilvl ][ index ].add_interval( interval ); + diff_ncells += interval.size(); + }); + + } + + // new mesh for current process + return Mesh_t( cl, minlevel, maxlevel ); + } + + } struct MPI_Load_Balance { int32_t _load; From 625c4cf4cfb4b2128d7a67cca2556e5d50c0d3b0 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 17 Apr 2024 11:13:05 +0200 Subject: [PATCH 047/170] fix SFC load balance --- include/samurai/load_balancing_sfc.hpp | 73 +++++++++++++++++++++----- 1 file changed, 61 insertions(+), 12 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index bdd68b366..1f7c84a94 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -30,8 +30,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer - void load_balance_impl( Mesh & mesh ){ + template + void load_balance_impl( Mesh & mesh, Fields&... data ){ using Config = samurai::MRConfig; using Mesh_t = samurai::MRMesh; @@ -43,6 +43,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer> indices; + bool given; }; boost::mpi::communicator world; @@ -78,7 +79,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( icell( idim ) ); } - sfc_map[ _sfc.template getKey( ijk ) ] = { level, inter, index }; + sfc_map[ _sfc.template getKey( ijk ) ] = { level, inter, index, false }; ninterval ++; }); @@ -111,8 +112,15 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( ( my_load_i - load_interval[ static_cast( neighbour_rank_next ) ] ) * TRANSFER_PERCENT ); } - logs << "Neighbour prev : " << neighbour_rank_prev << ", loads : " << transfer_load_prev << std::endl; - logs << "Neighbour next : " << neighbour_rank_next << ", loads : " << transfer_load_next << std::endl; + logs << "Neighbour prev : " << neighbour_rank_prev << ", transfer of loads : " << transfer_load_prev << std::endl; + logs << "Neighbour next : " << neighbour_rank_next << ", transfer of loads : " << transfer_load_next << std::endl; + + /** + * In this section we update the current process mesh data based on what is sent and receive + * + */ + + CellList_t new_cl; // this will contains the final mesh of the current process // need send data to prev neighbour if( neighbour_rank_prev >= 0 && transfer_load_prev != 0 ){ @@ -125,6 +133,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer 0 ){ cl_to_send[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); + iter->second.given = true; // flag for "interval" has been sent my_load_i -= 1; transfer_load_prev += 1; niter_send ++; @@ -133,17 +142,30 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Sending " << niter_send << " interval to " << neighbour_rank_prev << std::endl; - CellArray_t ca_to_send = { cl_to_send, false }; + + logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_prev << std::endl; + world.send( neighbour_rank_prev, 42, ca_to_send ); - mesh.remove( ca_to_send ); + + // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); + // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; + + // mesh.remove( ca_to_send ); }else{ // need recv CellArray_t ca_to_rcv; world.recv( neighbour_rank_prev, 42, ca_to_rcv ); - mesh.merge( ca_to_rcv ); + + logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_prev << std::endl; + + // mesh.merge( ca_to_rcv ); + // add to CL what we just receive + samurai::for_each_interval( ca_to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ + new_cl[ level ][ index ].add_interval( interval ); + }); + } } @@ -158,6 +180,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer 0 ){ cl_to_send[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); + iter->second.given = true; my_load_i -= 1; transfer_load_next += 1; niter_send ++; @@ -166,21 +189,47 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Sending " << niter_send << " interval to " << neighbour_rank_next << std::endl; - CellArray_t ca_to_send = { cl_to_send, false }; + + logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_next << std::endl; + world.send( neighbour_rank_next, 42, ca_to_send ); + + // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); + // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; + mesh.remove( ca_to_send ); }else{ // need recv CellArray_t ca_to_rcv; world.recv( neighbour_rank_next, 42, ca_to_rcv ); - mesh.merge( ca_to_rcv ); + + logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_next << std::endl; + + // mesh.merge( ca_to_rcv ); + // add to CL what we just receive + samurai::for_each_interval( ca_to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ + new_cl[ level ][ index ].add_interval( interval ); + }); + } } + // last loop over the map to add what is left + for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) { + if( ! iter->second.given ){ + new_cl[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); + } + } + + Mesh_t new_mesh = { new_cl, false }; + + // new_cl is built + + // NOW build a new MRMesh. This require to dev. a new constructor with the discover_neighbour ? + // update neighbour mesh - this should end up with the same result but .. mesh.update_mesh_neighbour(); From 74f7b621fbd5426e6221a418b6cfee0f3c6169f9 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Wed, 17 Apr 2024 20:15:22 +0200 Subject: [PATCH 048/170] add mesh constructor with mpi neighbourhood --- include/samurai/mesh.hpp | 21 +++++++++++++++++++++ include/samurai/mr/mesh.hpp | 14 ++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index dfe18c242..f4c48dacb 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -159,6 +159,10 @@ namespace samurai double approx_box_tol = lca_type::default_approx_box_tol, double scaling_factor = 0); + // Used for load balancing + Mesh_base( const cl_type & cl, std::size_t min_level, std::size_t max_level, + std::vector & neighbourhood ); + derived_type& derived_cast() & noexcept; const derived_type& derived_cast() const& noexcept; derived_type derived_cast() && noexcept; @@ -344,6 +348,23 @@ namespace samurai renumbering(); } + template + inline Mesh_base::Mesh_base( const cl_type & cl, std::size_t min_level, std::size_t max_level, + std::vector & neighbourhood ): m_min_level( min_level), m_max_level( max_level), + m_mpi_neighbourhood( neighbourhood ){ + + // What to do with m_periodic ? + // what to do with m_domain ? + + m_cells[mesh_id_t::cells] = {cl, false}; + + update_mesh_neighbour(); // required to do that here ?? + construct_subdomain(); // required ? + construct_union(); // required ? + update_sub_mesh(); // perform MPI allReduce calls + renumbering(); // required ? + } + template inline auto Mesh_base::cells() -> mesh_t& { diff --git a/include/samurai/mr/mesh.hpp b/include/samurai/mr/mesh.hpp index f5b0daf44..67c9b4d47 100644 --- a/include/samurai/mr/mesh.hpp +++ b/include/samurai/mr/mesh.hpp @@ -86,6 +86,13 @@ namespace samurai const std::array& periodic, double approx_box_tol = lca_type::default_approx_box_tol, double scaling_factor = 0); + // Used for load balancing + MRMesh(const cl_type & cl, + std::size_t min_level, + std::size_t max_level, + std::vector & neighbourhood, + double approx_box_tol = lca_type::default_approx_box_tol, + double scaling_factor = 0); void update_sub_mesh_impl(); @@ -132,6 +139,13 @@ namespace samurai { } + template + inline MRMesh::MRMesh( const cl_type & cl, std::size_t min_level, std::size_t max_level, + std::vector & neighbourhood) + : base_type(cl, min_level, max_level, neighbourhood ) + { + } + template inline void MRMesh::update_sub_mesh_impl() { From e705e252ccfad3fb6274c17a524a158c800f8cb0 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Wed, 17 Apr 2024 20:15:51 +0200 Subject: [PATCH 049/170] return new mesh from load balancing with sfc --- demos/FiniteVolume/advection_2d.cpp | 4 ++-- include/samurai/load_balancing.hpp | 4 ++-- include/samurai/load_balancing_sfc.hpp | 26 ++++++++++---------------- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 3b24e1c1f..f97af6deb 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -198,7 +198,7 @@ int main(int argc, char* argv[]) // Multiresolution parameters std::size_t min_level = 4; - std::size_t max_level = 10; + std::size_t max_level = 6; double mr_epsilon = 2.e-4; // Threshold used by multiresolution double mr_regularity = 1.; // Regularity guess for multiresolution bool correction = false; @@ -261,7 +261,7 @@ int main(int argc, char* argv[]) if( nt % 20 == 0 && nt > 1 ){ std::cout << "\t> Load balancing mesh ... " << std::endl; myTimers.start("load-balancing"); - balancer.load_balance( mesh ); + auto new_mesh = balancer.load_balance( mesh ); myTimers.stop("load-balancing"); unp1.resize(); diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 6f83c80ef..c5e5386d5 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -251,8 +251,8 @@ namespace samurai{ public: template - void load_balance( Mesh & mesh, Args&... kw ){ - static_cast(this)->load_balance_impl( mesh, kw... ); + Mesh load_balance( Mesh & mesh, Args&... kw ){ + return static_cast(this)->load_balance_impl( mesh, kw... ); } template diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 1f7c84a94..abc4c9955 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -31,7 +31,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer - void load_balance_impl( Mesh & mesh, Fields&... data ){ + Mesh load_balance_impl( Mesh & mesh, Fields&... data ){ using Config = samurai::MRConfig; using Mesh_t = samurai::MRMesh; @@ -86,10 +86,9 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancerfirst, sfc_map.rbegin()->first }; - - logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; + // Key boundaries of current process - unused for now + // SFC_key_t interval[ 2 ] = { sfc_map.begin()->first, sfc_map.rbegin()->first }; + // logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; std::vector load_interval; int my_load_i = static_cast( samurai::cmptLoad( mesh ) ); @@ -224,20 +223,15 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( mesh ); - samurai::discover_neighbour( mesh ); + samurai::discover_neighbour( new_mesh ); + samurai::discover_neighbour( new_mesh ); + + return new_mesh; } From 9fc92a9866d212523abaeda939916c48281fcbf2 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 18 Apr 2024 17:05:07 +0200 Subject: [PATCH 050/170] fix update neighbour in constructor --- include/samurai/mesh.hpp | 118 +++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index f4c48dacb..ecb2bb587 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -133,9 +133,8 @@ namespace samurai void update_mesh_neighbour(); void to_stream(std::ostream& os) const; - - void merge( ca_type & lca ); - void remove( ca_type & lca ); + void merge(ca_type& lca); + void remove(ca_type& lca); protected: @@ -143,7 +142,7 @@ namespace samurai Mesh_base() = default; // cppcheck-suppress uninitMemberVar Mesh_base(const cl_type& cl, const self_type& ref_mesh); - Mesh_base( const ca_type &ca, const self_type & ref_mesh); + Mesh_base(const ca_type& ca, const self_type& ref_mesh); Mesh_base(const cl_type& cl, std::size_t min_level, std::size_t max_level); Mesh_base(const samurai::Box& b, std::size_t start_level, @@ -160,8 +159,7 @@ namespace samurai double scaling_factor = 0); // Used for load balancing - Mesh_base( const cl_type & cl, std::size_t min_level, std::size_t max_level, - std::vector & neighbourhood ); + Mesh_base(const cl_type& cl, std::size_t min_level, std::size_t max_level, std::vector& neighbourhood); derived_type& derived_cast() & noexcept; const derived_type& derived_cast() const& noexcept; @@ -200,11 +198,11 @@ namespace samurai { ar& m_cells[id]; } - ar& m_domain; - ar& m_subdomain; - ar& m_union; - ar& m_min_level; - ar& m_min_level; + ar & m_domain; + ar & m_subdomain; + ar & m_union; + ar & m_min_level; + ar & m_min_level; } #endif }; @@ -341,28 +339,35 @@ namespace samurai { m_cells[mesh_id_t::cells] = ca; - update_mesh_neighbour(); construct_subdomain(); construct_union(); update_sub_mesh(); renumbering(); + update_mesh_neighbour(); } template - inline Mesh_base::Mesh_base( const cl_type & cl, std::size_t min_level, std::size_t max_level, - std::vector & neighbourhood ): m_min_level( min_level), m_max_level( max_level), - m_mpi_neighbourhood( neighbourhood ){ - - // What to do with m_periodic ? - // what to do with m_domain ? - + inline Mesh_base::Mesh_base(const cl_type& cl, + std::size_t min_level, + std::size_t max_level, + std::vector& neighbourhood) + : m_min_level(min_level) + , m_max_level(max_level) + , m_mpi_neighbourhood(neighbourhood) + { + m_periodic.fill(false); + assert(min_level <= max_level); + + // what to do with m_domain ? + m_domain = m_subdomain; + m_cells[mesh_id_t::cells] = {cl, false}; - update_mesh_neighbour(); // required to do that here ?? - construct_subdomain(); // required ? - construct_union(); // required ? - update_sub_mesh(); // perform MPI allReduce calls - renumbering(); // required ? + update_mesh_neighbour(); // required to do that here ?? + construct_subdomain(); // required ? + construct_union(); // required ? + update_sub_mesh(); // perform MPI allReduce calls + renumbering(); // required ? } template @@ -483,8 +488,9 @@ namespace samurai template template - inline auto Mesh_base::get_interval(std::size_t level, const interval_t& interval, const xt::xexpression& index) const - -> const interval_t& + inline auto Mesh_base::get_interval(std::size_t level, + const interval_t& interval, + const xt::xexpression& index) const -> const interval_t& { return m_cells[mesh_id_t::reference].get_interval(level, interval, index); } @@ -691,7 +697,8 @@ namespace samurai } template - void Mesh_base::partition_mesh([[maybe_unused]] std::size_t start_level, [[maybe_unused]] const Box& global_box) { + void Mesh_base::partition_mesh([[maybe_unused]] std::size_t start_level, [[maybe_unused]] const Box& global_box) + { #ifdef SAMURAI_WITH_MPI mpi::communicator world; auto rank = world.rank(); @@ -710,7 +717,7 @@ namespace samurai int product_of_sizes = 1; for (std::size_t d = 0; d < dim - 1; ++d) { - sizes[d] = std::max( static_cast(floor(pow(size, 1. / dim) * global_box.length()[d] / length_harmonic_avg)), 1 ); + sizes[d] = std::max(static_cast(floor(pow(size, 1. / dim) * global_box.length()[d] / length_harmonic_avg)), 1); product_of_sizes *= sizes[d]; } @@ -778,8 +785,8 @@ namespace samurai this->m_cells[mesh_id_t::cells][start_level] = subdomain_cells; // end comment for git rebase - **/ this->m_cells[mesh_id_t::cells][start_level] = {start_level, subdomain_box}; + **/ m_mpi_neighbourhood.reserve(static_cast(size) - 1); for (int ir = 0; ir < size; ++ir) @@ -819,52 +826,57 @@ namespace samurai m_mpi_neighbourhood.push_back(neighbour(shift)); } }); - + #endif } template - void Mesh_base::merge( ca_type & lca ) { + void Mesh_base::merge(ca_type& lca) + { // merge received cells - auto & refmesh = this->m_cells[ mesh_id_t::cells ]; + auto& refmesh = this->m_cells[mesh_id_t::cells]; - auto minlevel = std::min( refmesh.min_level(), lca.min_level() ); - auto maxlevel = std::max( refmesh.max_level(), lca.max_level() ); + auto minlevel = std::min(refmesh.min_level(), lca.min_level()); + auto maxlevel = std::max(refmesh.max_level(), lca.max_level()); cl_type cl; - for( size_t ilvl=minlevel; ilvl<=maxlevel; ++ilvl ) { - - auto un = samurai::union_( refmesh[ ilvl ], lca[ ilvl ] ); + for (size_t ilvl = minlevel; ilvl <= maxlevel; ++ilvl) + { + auto un = samurai::union_(refmesh[ilvl], lca[ilvl]); - un([&]( auto & interval, auto & indices ) { - cl[ ilvl ][ indices ].add_interval( interval ); - }); + un( + [&](auto& interval, auto& indices) + { + cl[ilvl][indices].add_interval(interval); + }); } - refmesh = { cl, false }; + refmesh = {cl, false}; } template - void Mesh_base::remove( ca_type & lca) { - auto & refmesh = this->m_cells[ mesh_id_t::cells ]; + void Mesh_base::remove(ca_type& lca) + { + auto& refmesh = this->m_cells[mesh_id_t::cells]; - // remove cells + // remove cells cl_type cl; size_t diff_ncells = 0; - for( size_t ilvl=refmesh.min_level(); ilvl<=refmesh.max_level(); ++ilvl ) { - - auto diff = samurai::difference( refmesh[ ilvl ], lca[ ilvl ] ); - - diff([&]( auto & interval, auto & index ) { - cl[ ilvl ][ index ].add_interval( interval ); - diff_ncells += interval.size(); - }); + for (size_t ilvl = refmesh.min_level(); ilvl <= refmesh.max_level(); ++ilvl) + { + auto diff = samurai::difference(refmesh[ilvl], lca[ilvl]); + diff( + [&](auto& interval, auto& index) + { + cl[ilvl][index].add_interval(interval); + diff_ncells += interval.size(); + }); } // new mesh for current process - refmesh = { cl, false }; + refmesh = {cl, false}; } template From a5b3764af19ccf9d782aded871e69a85da176b5e Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 18 Apr 2024 17:05:25 +0200 Subject: [PATCH 051/170] update load balancing SFC with data exchange --- include/samurai/load_balancing.hpp | 1167 ++++++++++++++++------------ 1 file changed, 686 insertions(+), 481 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index c5e5386d5..3d3193f0d 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -1,141 +1,176 @@ #pragma once -#include -#include #include #include +#include +#include #include +#include #include -#include #include -#include +#include -// statistics +// statistics #include -namespace samurai{ +namespace samurai +{ + + namespace load_balance + { - namespace load_balance { - template - Mesh_t merge( Mesh_t & mesh, const CellArray_t & lca ) { + Mesh_t merge(Mesh_t& mesh, const CellArray_t& lca) + { using cl_type = typename Mesh_t::cl_type; - auto & refmesh = mesh[ Mesh_t::mesh_id_t::cells ]; + auto& refmesh = mesh[Mesh_t::mesh_id_t::cells]; - auto minlevel = std::min( refmesh.min_level(), lca.min_level() ); - auto maxlevel = std::max( refmesh.max_level(), lca.max_level() ); + auto minlevel = std::min(refmesh.min_level(), lca.min_level()); + auto maxlevel = std::max(refmesh.max_level(), lca.max_level()); cl_type cl; - for( size_t ilvl=minlevel; ilvl<=maxlevel; ++ilvl ) { - - auto un = samurai::union_( refmesh[ ilvl ], lca[ ilvl ] ); + for (size_t ilvl = minlevel; ilvl <= maxlevel; ++ilvl) + { + auto un = samurai::union_(refmesh[ilvl], lca[ilvl]); - un([&]( auto & interval, auto & indices ) { - cl[ ilvl ][ indices ].add_interval( interval ); - }); + un( + [&](auto& interval, auto& indices) + { + cl[ilvl][indices].add_interval(interval); + }); } - return Mesh_t( cl, minlevel, maxlevel ); - + return Mesh_t(cl, minlevel, maxlevel); } template - Mesh_t remove( Mesh_t & mesh, CellArray_t & lca ) { + Mesh_t remove(Mesh_t& mesh, CellArray_t& lca) + { using cl_type = typename Mesh_t::cl_type; - auto & refmesh = mesh[ Mesh_t::mesh_id_t::cells ]; + auto& refmesh = mesh[Mesh_t::mesh_id_t::cells]; - auto minlevel = std::min( refmesh.min_level(), lca.min_level() ); - auto maxlevel = std::max( refmesh.max_level(), lca.max_level() ); + auto minlevel = std::min(refmesh.min_level(), lca.min_level()); + auto maxlevel = std::max(refmesh.max_level(), lca.max_level()); - // remove cells + // remove cells cl_type cl; size_t diff_ncells = 0; - for( size_t ilvl=minlevel; ilvl<=maxlevel; ++ilvl ) { - - auto diff = samurai::difference( refmesh[ ilvl ], lca[ ilvl ] ); - - diff([&]( auto & interval, auto & index ) { - cl[ ilvl ][ index ].add_interval( interval ); - diff_ncells += interval.size(); - }); + for (size_t ilvl = minlevel; ilvl <= maxlevel; ++ilvl) + { + auto diff = samurai::difference(refmesh[ilvl], lca[ilvl]); + diff( + [&](auto& interval, auto& index) + { + cl[ilvl][index].add_interval(interval); + diff_ncells += interval.size(); + }); } // new mesh for current process - return Mesh_t( cl, minlevel, maxlevel ); + return Mesh_t(cl, minlevel, maxlevel); } } - - struct MPI_Load_Balance { + + struct MPI_Load_Balance + { int32_t _load; std::vector neighbour; std::vector load; std::vector fluxes; }; - enum Distance_t { L1, L2, LINF, GRAVITY }; - enum Direction_t { FACE, DIAG, FACE_AND_DIAG }; - enum BalanceElement_t { CELL, INTERVAL }; + enum Distance_t + { + L1, + L2, + LINF, + GRAVITY + }; + + enum Direction_t + { + FACE, + DIAG, + FACE_AND_DIAG + }; + + enum BalanceElement_t + { + CELL, + INTERVAL + }; /** * Compute distance base on different norm. - */ + */ - template - static inline double distance_l2( const Coord_t & d1, const Coord_t & d2 ) { + template + static inline double distance_l2(const Coord_t& d1, const Coord_t& d2) + { double dist = 0.; - for( size_t idim=0; idim < static_cast( dim ); ++idim){ - double d = d1( idim ) - d2( idim ); + for (size_t idim = 0; idim < static_cast(dim); ++idim) + { + double d = d1(idim) - d2(idim); dist += d * d; } - return std::sqrt( dist ); + return std::sqrt(dist); } - template - static inline double distance_inf( const Coord_t & d1, const Coord_t & d2 ) { + template + static inline double distance_inf(const Coord_t& d1, const Coord_t& d2) + { double dist = 0.; - for( size_t idim=0; idim < static_cast( dim ); ++idim){ - dist = std::max( std::abs( d1( idim ) - d2( idim ) ), dist ) ; + for (size_t idim = 0; idim < static_cast(dim); ++idim) + { + dist = std::max(std::abs(d1(idim) - d2(idim)), dist); } return dist; } - template - static inline double distance_l1( const Coord_t & d1, const Coord_t & d2 ) { + template + static inline double distance_l1(const Coord_t& d1, const Coord_t& d2) + { double dist = 0.; - for( size_t idim=0; idim < static_cast( dim ); ++idim ){ - dist += std::abs( d1( idim ) - d2( idim ) ) ; + for (size_t idim = 0; idim < static_cast(dim); ++idim) + { + dist += std::abs(d1(idim) - d2(idim)); } return dist; } /** * Compute the load of the current process based on intervals or cells. It uses the - * mesh_id_t::cells to only consider leaves. - */ - template - static std::size_t cmptLoad( const Mesh_t & mesh ) { - + * mesh_id_t::cells to only consider leaves. + */ + template + static std::size_t cmptLoad(const Mesh_t& mesh) + { using mesh_id_t = typename Mesh_t::mesh_id_t; - const auto & current_mesh = mesh[ mesh_id_t::cells ]; + const auto& current_mesh = mesh[mesh_id_t::cells]; std::size_t current_process_load = 0; - - if constexpr ( elem == BalanceElement_t::CELL ) { + + if constexpr (elem == BalanceElement_t::CELL) + { // cell-based load without weight. - samurai::for_each_interval( current_mesh, - [&]( [[maybe_unused]] std::size_t level, const auto& interval, [[maybe_unused]] const auto& index ){ - current_process_load += interval.size(); - }); - } else { + samurai::for_each_interval(current_mesh, + [&]([[maybe_unused]] std::size_t level, const auto& interval, [[maybe_unused]] const auto& index) + { + current_process_load += interval.size(); + }); + } + else + { // interval-based load without weight - for( std::size_t level=current_mesh.min_level(); level<=current_mesh.max_level(); ++level ){ - current_process_load += current_mesh[ level ].shape()[ 0 ]; // only in x-axis ; + for (std::size_t level = current_mesh.min_level(); level <= current_mesh.max_level(); ++level) + { + current_process_load += current_mesh[level].shape()[0]; // only in x-axis ; } } @@ -143,31 +178,31 @@ namespace samurai{ } /** - * Compute fluxes based on load computing stategy based on graph with label + * Compute fluxes based on load computing stategy based on graph with label * propagation algorithm. Return, for the current process, the flux in term of * load, i.e. the quantity of "load" to transfer to its neighbours. If the load * is negative, it means that the process (current) must send load to neighbour, * if positive it means that it must receive load. - * + * * This function use 2 MPI all_gather calls. - * - */ - template - std::vector cmptFluxes( Mesh_t & mesh ) { - + * + */ + template + std::vector cmptFluxes(Mesh_t& mesh) + { using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; boost::mpi::communicator world; // give access to geometricaly neighbour process rank and mesh - std::vector & neighbourhood = mesh.mpi_neighbourhood(); - size_t n_neighbours = neighbourhood.size(); + std::vector& neighbourhood = mesh.mpi_neighbourhood(); + size_t n_neighbours = neighbourhood.size(); // load of current process - int my_load = static_cast( cmptLoad( mesh ) ); + int my_load = static_cast(cmptLoad(mesh)); // fluxes between processes - std::vector fluxes( n_neighbours, 0); + std::vector fluxes(n_neighbours, 0); // load of each process (all processes not only neighbours) std::vector loads; @@ -181,140 +216,290 @@ namespace samurai{ // compute updated my_load for current process based on its neighbourhood int my_load_new = my_load; - for( std::size_t n_i = 0; n_i < n_neighbours; ++n_i ){ + for (std::size_t n_i = 0; n_i < n_neighbours; ++n_i) + { + std::size_t neighbour_rank = static_cast(neighbourhood[n_i].rank); + int neighbour_load = loads[neighbour_rank]; + double diff_load = static_cast(neighbour_load - my_load); - std::size_t neighbour_rank = static_cast( neighbourhood[ n_i ].rank ); - int neighbour_load = loads[ neighbour_rank ]; - double diff_load = static_cast( neighbour_load - my_load ); + std::size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[neighbour_rank]; - std::size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[ neighbour_rank ]; + double weight = 1. / static_cast(std::max(n_neighbours, nb_neighbours_neighbour) + 1); - double weight = 1. / static_cast( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); + int transfertLoad = static_cast(std::lround(weight * diff_load)); - int transfertLoad = static_cast( std::lround( weight * diff_load ) ); - - fluxes[ n_i ] += transfertLoad; + fluxes[n_i] += transfertLoad; my_load_new += transfertLoad; } // Cannot have a negative or null load on an MPI process ! - assert( my_load_new > 0 ); + assert(my_load_new > 0); return fluxes; + } - } - - - /** Is gravity the key ? That would be awesome =) - * This is not a distance but well does not change anything to the algorithm - * - G m m' / R ? - * - * What should be m ? m' ? Let's G = 1. - * - **/ - template - static inline double gravity( const Coord_t & d1, const Coord_t & d2 ) { - double dist = distance_l2( d1, d2 ); + /** Is gravity the key ? That would be awesome =) + * This is not a distance but well does not change anything to the algorithm + * - G m m' / R ? + * + * What should be m ? m' ? Let's G = 1. + * + **/ + template + static inline double gravity(const Coord_t& d1, const Coord_t& d2) + { + double dist = distance_l2(d1, d2); // makes high level ( smallest ) cells to be exchanged more easily. 10 here is max // level, hardcoded for tests. // double m = 1./ static_cast( 1 << ( 10 - d1.level ) ) , m_p = 1.; - + // makes bigger cells to be exchanged more easily // double m = 1./ static_cast( 1 << d1.level ) , m_p = 1.; constexpr double G = 1.; double m = 1., m_p = 1.; - double f = - G * m * m_p / ( dist * dist ); + double f = -G * m * m_p / (dist * dist); return f; } // we are using Cell_t to allow ponderation using level; - template - inline constexpr double getDistance( const Coord_t & cc, const Coord_t & d ){ - static_assert( dim == 2 || dim == 3 ); - if constexpr ( dist == Distance_t::L1 ){ - return distance_l1( cc, d ); - }else if constexpr( dist == Distance_t::L2 ) { - return distance_l2( cc, d ); - }else if constexpr( dist == Distance_t::LINF ) { - return distance_inf( cc, d ); - }else if constexpr( dist == Distance_t::GRAVITY ){ - return gravity( cc, d ); + template + inline constexpr double getDistance(const Coord_t& cc, const Coord_t& d) + { + static_assert(dim == 2 || dim == 3); + if constexpr (dist == Distance_t::L1) + { + return distance_l1(cc, d); + } + else if constexpr (dist == Distance_t::L2) + { + return distance_l2(cc, d); + } + else if constexpr (dist == Distance_t::LINF) + { + return distance_inf(cc, d); + } + else if constexpr (dist == Distance_t::GRAVITY) + { + return gravity(cc, d); } } - template - class LoadBalancer { - - public: + template + class LoadBalancer + { + private: - template - Mesh load_balance( Mesh & mesh, Args&... kw ){ - return static_cast(this)->load_balance_impl( mesh, kw... ); - } + template + void update_field(Mesh_t& new_mesh, Field_t& field) const + { + using mesh_id_t = typename Mesh_t::mesh_id_t; + using value_t = typename Field_t::value_type; + boost::mpi::communicator world; + + Field_t new_field("new_f", new_mesh); - template - void evaluate_balancing( Mesh & mesh ) const { +#ifdef SAMURAI_CHECK_NAN + new_field.fill(std::nan("")); +#else + new_field.fill(0); +#endif - boost::mpi::communicator world; + auto& mesh = field.mesh(); - if( world.rank() == 0 ) SAMURAI_TRACE( "[LoadBalancer::evaluate_balancing]::Entering function ... " ); + // auto min_level = boost::mpi::all_reduce(world, mesh[mesh_id_t::cells].min_level(), boost::mpi::minimum()); + // auto max_level = boost::mpi::all_reduce(world, mesh[mesh_id_t::cells].max_level(), boost::mpi::maximum()); - std::vector load_cells, load_interval; + auto min_level = mesh.min_level(); + auto max_level = mesh.max_level(); + + for (std::size_t level = min_level; level <= max_level; ++level) + { + auto set = intersection(mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + set.apply_op(copy(new_field, field)); + } + + std::vector req; + std::vector> to_send(new_mesh.mpi_neighbourhood().size()); + + std::size_t i_neigh = 0; + + for (auto& neighbour : new_mesh.mpi_neighbourhood()) + { + for (std::size_t level = min_level; level <= max_level; ++level) { - int my_load_i = static_cast( cmptLoad( mesh ) ); - boost::mpi::all_gather( world, my_load_i, load_interval ); - - int my_load_c = static_cast( cmptLoad( mesh ) ); - boost::mpi::all_gather( world, my_load_c, load_cells ); + if (!mesh[mesh_id_t::cells][level].empty() && !neighbour.mesh[mesh_id_t::cells][level].empty()) + { + auto out_interface = intersection(mesh[mesh_id_t::cells][level], neighbour.mesh[mesh_id_t::cells][level]); + out_interface( + [&](const auto& i, const auto& index) + { + std::copy(field(level, i, index).begin(), field(level, i, index).end(), std::back_inserter(to_send[i_neigh])); + // std::cerr << fmt::format("Process {}, send interval {}", world.rank(), i) << std::endl; + }); + } } - if( world.rank() == 0 ){ - std::cerr << "\t> LoadBalancer statistics : " << std::endl; + if (to_send[i_neigh].size() != 0) + { + req.push_back(world.isend(neighbour.rank, neighbour.rank, to_send[i_neigh++])); + } + } - std::vector::iterator min_load = std::min_element( load_cells.begin(), load_cells.end()); - auto rank = std::distance( load_cells.begin(), min_load ); - std::cerr << "\t\t> Min load {" << * min_load << ", cells } @ rank # " << rank << std::endl; + for (auto& neighbour : mesh.mpi_neighbourhood()) + { + bool isintersect = false; + for (std::size_t level = min_level; level <= max_level; ++level) + { + if (!new_mesh[mesh_id_t::cells][level].empty() && !neighbour.mesh[mesh_id_t::cells][level].empty()) + { + std::vector to_recv; + std::ptrdiff_t count = 0; - std::vector::iterator max_load = std::max_element( load_cells.begin(), load_cells.end()); - rank = std::distance( load_cells.begin(), max_load ); - std::cerr << "\t\t> Max load {" << * max_load << ", cells } @ rank # " << rank << std::endl; + auto in_interface = intersection(neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + in_interface( + [&](const auto& i, const auto& index) + { + isintersect = true; + }); + if (isintersect) + { + break; + } + } + } + if (isintersect) + { + std::ptrdiff_t count = 0; + std::vector to_recv; + world.recv(neighbour.rank, world.rank(), to_recv); + + for (std::size_t level = min_level; level <= max_level; ++level) + { + if (!new_mesh[mesh_id_t::cells][level].empty() && !neighbour.mesh[mesh_id_t::cells][level].empty()) + { + auto in_interface = intersection(neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + + in_interface( + [&](const auto& i, const auto& index) + { + std::copy(to_recv.begin() + count, + to_recv.begin() + count + static_cast(i.size()), + new_field(level, i, index).begin()); + count += static_cast(i.size()); + + // std::cerr << fmt::format("Process {}, recv interval {}", world.rank(), i) << std::endl; + }); + } + } } + } + + if (!req.empty()) + { + mpi::wait_all(req.begin(), req.end()); + } - std::string _stats = fmt::format( "statistics_process_{}", world.rank() ); - samurai::Statistics s ( _stats ); + std::swap(field.array(), new_field.array()); + } + + template + void update_fields(Mesh_t& new_mesh, Field_t& field, Fields_t&... kw) const + + { + update_field(new_mesh, field); + update_fields(new_mesh, kw...); + } + + template + void update_fields(Mesh_t& new_mesh) const + { + } + + public: - s( "stats", mesh ); - - // s( "statistics" , mesh ); + template + void load_balance(Mesh& mesh, Field_t& field, Fields&... kw) + { + // build new mesh using "Flavor" load balancing strategy + auto new_mesh = static_cast(this)->load_balance_impl(mesh); - // no need to implement this in derived class, could be general. - // - // static_cast(this)->evaluate_impl( mesh, kw... ); + // Manage physical fields on new load balanced mesh + update_fields(new_mesh, field, kw...); + + // std::swap(mesh, new_mesh); + + // swapping meshes + field.mesh().swap(new_mesh); + // save("load_balance", mesh, field); + } + + template + void evaluate_balancing(Mesh& mesh) const + { + boost::mpi::communicator world; + + if (world.rank() == 0) + { + SAMURAI_TRACE("[LoadBalancer::evaluate_balancing]::Entering function ... "); } + std::vector load_cells, load_interval; + { + int my_load_i = static_cast(cmptLoad(mesh)); + boost::mpi::all_gather(world, my_load_i, load_interval); + + int my_load_c = static_cast(cmptLoad(mesh)); + boost::mpi::all_gather(world, my_load_c, load_cells); + } + + if (world.rank() == 0) + { + std::cerr << "\t> LoadBalancer statistics : " << std::endl; + + std::vector::iterator min_load = std::min_element(load_cells.begin(), load_cells.end()); + auto rank = std::distance(load_cells.begin(), min_load); + std::cerr << "\t\t> Min load {" << *min_load << ", cells } @ rank # " << rank << std::endl; + + std::vector::iterator max_load = std::max_element(load_cells.begin(), load_cells.end()); + rank = std::distance(load_cells.begin(), max_load); + std::cerr << "\t\t> Max load {" << *max_load << ", cells } @ rank # " << rank << std::endl; + } + + std::string _stats = fmt::format("statistics_process_{}", world.rank()); + samurai::Statistics s(_stats); + + s("stats", mesh); + + // s( "statistics" , mesh ); + + // no need to implement this in derived class, could be general. + // + // static_cast(this)->evaluate_impl( mesh, kw... ); + } }; /** * Precomputed direction to face-to-face element for both 3D and 2D - */ - template - constexpr auto getDirectionFace() { - + */ + template + constexpr auto getDirectionFace() + { using base_stencil = xt::xtensor_fixed>; - xt::xtensor_fixed> stencils; - std::size_t nstencils = static_cast( 2 * dim ); - for( std::size_t ist=0; ist> stencils; + std::size_t nstencils = static_cast(2 * dim); + for (std::size_t ist = 0; ist < nstencils; ++ist) + { + stencils[ist].fill(0); + stencils[ist][ist / 2] = 1 - (ist % 2) * 2; } - // if constexpr( dim == 2 ){ + // if constexpr( dim == 2 ){ // return xt::xtensor_fixed> {{{1, 0}, {-1, 0}, {0, 1}, {0, -1}}}; // } @@ -323,136 +508,131 @@ namespace samurai{ /** * Precomputed direction to diagonal element for both 3D and 2D cases. - */ - template - constexpr auto getDirectionDiag() { - + */ + template + constexpr auto getDirectionDiag() + { using base_stencil = xt::xtensor_fixed>; - static_assert( dim == 2 || dim == 3 ); + static_assert(dim == 2 || dim == 3); - if constexpr( dim == 2 ){ - return xt::xtensor_fixed> {{{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}}; + if constexpr (dim == 2) + { + return xt::xtensor_fixed>{ + {{1, 1}, {1, -1}, {-1, 1}, {-1, -1}} + }; } - if constexpr ( dim == 3 ){ - return xt::xtensor_fixed> { - {{ 1, 1, -1}, { 1, -1, -1}, - {-1, 1, -1}, {-1, -1, -1}, - { 1, 0, -1}, { -1, 0, -1}, - { 0, 1, -1}, { 0, -1, -1}, - { 1, 1, 0}, { 1, -1, 0}, - {-1, 1, 0}, {-1, -1, 0}, - { 1, 1, 1}, { 1, -1, 1}, - {-1, 1, 1}, {-1, -1, 1}, - { 1, 0, 1}, { -1, 0, 1}, - { 0, 1, 1}, { 0, -1, 1}}}; + if constexpr (dim == 3) + { + return xt::xtensor_fixed>{ + {{1, 1, -1}, {1, -1, -1}, {-1, 1, -1}, {-1, -1, -1}, {1, 0, -1}, {-1, 0, -1}, {0, 1, -1}, + {0, -1, -1}, {1, 1, 0}, {1, -1, 0}, {-1, 1, 0}, {-1, -1, 0}, {1, 1, 1}, {1, -1, 1}, + {-1, 1, 1}, {-1, -1, 1}, {1, 0, 1}, {-1, 0, 1}, {0, 1, 1}, {0, -1, 1}} + }; } - } /** * Precompute direction to element in all direction face + diagonals: 26 in 3D, 8 in 2D - */ - template - constexpr auto getDirectionFaceAndDiag() { - + */ + template + constexpr auto getDirectionFaceAndDiag() + { using base_stencil = xt::xtensor_fixed>; - static_assert( dim == 2 || dim == 3 ); + static_assert(dim == 2 || dim == 3); - if constexpr( dim == 2 ){ - return xt::xtensor_fixed> {{{-1, 0}, { 1, 0}, - { 1, 1}, { 1, -1}, - { 0, -1}, { 0, 1}, - {-1, 1}, {-1, -1}}}; + if constexpr (dim == 2) + { + return xt::xtensor_fixed>{ + {{-1, 0}, {1, 0}, {1, 1}, {1, -1}, {0, -1}, {0, 1}, {-1, 1}, {-1, -1}} + }; } - if constexpr ( dim == 3 ){ - return xt::xtensor_fixed> { - {{ 1, 0, 0}, {-1, 0, 0}, - { 0, 1, 0}, { 0, -1, 0}, - { 0, 0, 1}, { 0, 0, -1}, - { 1, 1, -1}, { 1, -1, -1}, - {-1, 1, -1}, {-1, -1, -1}, - { 1, 0, -1}, { -1, 0, -1}, - { 0, 1, -1}, { 0, -1, -1}, - { 1, 1, 0}, { 1, -1, 0}, - {-1, 1, 0}, {-1, -1, 0}, - { 1, 1, 1}, { 1, -1, 1}, - {-1, 1, 1}, {-1, -1, 1}, - { 1, 0, 1}, { -1, 0, 1}, - { 0, 1, 1}, { 0, -1, 1}}}; + if constexpr (dim == 3) + { + return xt::xtensor_fixed>{ + {{1, 0, 0}, {-1, 0, 0}, {0, 1, 0}, {0, -1, 0}, {0, 0, 1}, {0, 0, -1}, {1, 1, -1}, {1, -1, -1}, {-1, 1, -1}, + {-1, -1, -1}, {1, 0, -1}, {-1, 0, -1}, {0, 1, -1}, {0, -1, -1}, {1, 1, 0}, {1, -1, 0}, {-1, 1, 0}, {-1, -1, 0}, + {1, 1, 1}, {1, -1, 1}, {-1, 1, 1}, {-1, -1, 1}, {1, 0, 1}, {-1, 0, 1}, {0, 1, 1}, {0, -1, 1}} + }; } - } - template - inline auto getDirection(){ - static_assert( dim == 2 || dim == 3 ); - if constexpr ( dir == Direction_t::FACE ){ + template + inline auto getDirection() + { + static_assert(dim == 2 || dim == 3); + if constexpr (dir == Direction_t::FACE) + { return getDirectionFace(); - }else if constexpr( dir == Direction_t::DIAG ) { + } + else if constexpr (dir == Direction_t::DIAG) + { return getDirectionDiag(); - }else if constexpr( dir == Direction_t::FACE_AND_DIAG ) { + } + else if constexpr (dir == Direction_t::FACE_AND_DIAG) + { return getDirectionFaceAndDiag(); } } /** * Compute the barycenter of the mesh given in parameter. - * - * Feat: adjust wght which is hardcoded to 1. for now. + * + * Feat: adjust wght which is hardcoded to 1. for now. * by passing in parameters an array for example to modulate * weight according to level - */ - template - xt::xtensor_fixed> _cmpCellBarycenter( Mesh_t & mesh ) { - + */ + template + xt::xtensor_fixed> _cmpCellBarycenter(Mesh_t& mesh) + { using Coord_t = xt::xtensor_fixed>; - + Coord_t bary; - bary.fill( 0. ); + bary.fill(0.); double wght_tot = 0.; - samurai::for_each_cell( mesh, [&]( auto & cell ){ - - // all cells equals - // constexpr double wght = 1.; + samurai::for_each_cell(mesh, + [&](auto& cell) + { + // all cells equals + // constexpr double wght = 1.; - // higher weight for large cells - // double wght = 1. / static_cast( 1 << cell.level ); + // higher weight for large cells + // double wght = 1. / static_cast( 1 << cell.level ); - // higher weight for small cells - double wght = 1. / static_cast( 1 << ( mesh.max_level() - cell.level ) ); + // higher weight for small cells + double wght = 1. / static_cast(1 << (mesh.max_level() - cell.level)); - auto cc = cell.center(); + auto cc = cell.center(); - for(int idim=0; idim - static auto cmptInterface( Mesh_t & mesh, Mesh_t & omesh ){ - + */ + template + static auto cmptInterface(Mesh_t& mesh, Mesh_t& omesh) + { using CellList_t = typename Mesh_t::cl_type; using CellArray_t = typename Mesh_t::ca_type; using mesh_id_t = typename Mesh_t::mesh_id_t; @@ -460,191 +640,204 @@ namespace samurai{ CellList_t interface; // operation are on leaves only - auto & currentMesh = mesh[ mesh_id_t::cells ]; - auto & otherMesh = omesh[ mesh_id_t::cells ]; + auto& currentMesh = mesh[mesh_id_t::cells]; + auto& otherMesh = omesh[mesh_id_t::cells]; // direction for translation to cmpt interface auto dirs = getDirection(); - for( size_t ist=0; ist( std::max( static_cast( currentMesh.min_level() ), - static_cast( level ) - static_cast( leveldiff ) ) ); - std::size_t maxlevel_check = std::min( currentMesh.max_level(), level + leveldiff ); - - for(size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + std::size_t minlevel_check = static_cast( + std::max(static_cast(currentMesh.min_level()), static_cast(level) - static_cast(leveldiff))); + std::size_t maxlevel_check = std::min(currentMesh.max_level(), level + leveldiff); + for (size_t projlevel = minlevel_check; projlevel <= maxlevel_check; ++projlevel) + { // translate neighbour from dir (hopefully to current) all direction are tested - auto set = translate( otherMesh[ level ], stencil ); - auto intersect = intersection( set, currentMesh[ projlevel ] ).on( projlevel ); + auto set = translate(otherMesh[level], stencil); + auto intersect = intersection(set, currentMesh[projlevel]).on(projlevel); size_t nbInter_ = 0, nbCells = 0; - intersect([&]( auto & interval, [[maybe_unused]] auto & index ) { - nbInter_ += 1; - nbCells += interval.size(); - }); - - // we get more interval / cells, than wanted because neighbour has bigger cells - // 2:1 balance required here if not, need another loop... - if( nbInter_ > 0 && projlevel > level ){ - static_assert( leveldiff == 1 ); // for now at least - auto set_ = translate( currentMesh[ projlevel ], stencil ); - auto diff_ = difference( intersect, set_ ); - - diff_([&]( auto & interval, auto & index ) { - interface[ projlevel ][ index ].add_interval( interval ); + intersect( + [&](auto& interval, [[maybe_unused]] auto& index) + { + nbInter_ += 1; + nbCells += interval.size(); }); - }else{ - if( nbInter_ > 0 ){ - intersect([&]( auto & interval, auto & index ) { - interface[ projlevel ][ index ].add_interval( interval ); + // we get more interval / cells, than wanted because neighbour has bigger cells + // 2:1 balance required here if not, need another loop... + if (nbInter_ > 0 && projlevel > level) + { + static_assert(leveldiff == 1); // for now at least + auto set_ = translate(currentMesh[projlevel], stencil); + auto diff_ = difference(intersect, set_); + + diff_( + [&](auto& interval, auto& index) + { + interface[projlevel][index].add_interval(interval); }); + } + else + { + if (nbInter_ > 0) + { + intersect( + [&](auto& interval, auto& index) + { + interface[projlevel][index].add_interval(interval); + }); } } - } - } - } - CellArray_t interface_ = { interface, false }; + CellArray_t interface_ = {interface, false}; return interface_; - } /** - * Compute cells at the interface between geometricaly adjacent + * Compute cells at the interface between geometricaly adjacent * domains. It relies on neighbourhood mpi_subdomain_t data structure * computed in Mesh_t. - * - * It returns a vector containing the number of cells (nc) and number of + * + * It returns a vector containing the number of cells (nc) and number of * intervals (ni) for each neighbour (0 to n), in that order. - * - * Example: nc_0, ni_0, nc_1, ni_1, ... - * - */ + * + * Example: nc_0, ni_0, nc_1, ni_1, ... + * + */ // template // void _computeCartesianInterface( Mesh_t & mesh, Field_t & field ){ - template - static auto _computeCartesianInterface( Mesh_t & mesh ){ - - using CellList_t = typename Mesh_t::cl_type; - using CellArray_t = typename Mesh_t::ca_type; + template + static auto _computeCartesianInterface(Mesh_t& mesh) + { + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; using mesh_id_t = typename Mesh_t::mesh_id_t; using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; // give access to geometricaly neighbour process rank and mesh - std::vector & neighbourhood = mesh.mpi_neighbourhood(); - std::size_t n_neighbours = neighbourhood.size(); + std::vector& neighbourhood = mesh.mpi_neighbourhood(); + std::size_t n_neighbours = neighbourhood.size(); - std::vector interface( n_neighbours ); + std::vector interface(n_neighbours); // operation are on leaves only - auto currentMesh = mesh[ mesh_id_t::cells ]; + auto currentMesh = mesh[mesh_id_t::cells]; // looks like using only FACE is better than using DIAG or FACE_AND_DIAG auto dirs = getDirection(); - - for( std::size_t nbi=0; nbi( std::max( static_cast( currentMesh.min_level() ), static_cast( level ) - 1 ) ); - std::size_t maxlevel_check = std::min( currentMesh.max_level() , level + static_cast( 1 ) ); - - for(std::size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + std::size_t minlevel_check = static_cast( + std::max(static_cast(currentMesh.min_level()), static_cast(level) - 1)); + std::size_t maxlevel_check = std::min(currentMesh.max_level(), level + static_cast(1)); + for (std::size_t projlevel = minlevel_check; projlevel <= maxlevel_check; ++projlevel) + { // translate neighbour from dir (hopefully to current) all direction are tested - auto set = translate( neighbour_mesh[ level ], stencil ); - auto intersect = intersection( set, currentMesh[ projlevel ] ).on( projlevel ); + auto set = translate(neighbour_mesh[level], stencil); + auto intersect = intersection(set, currentMesh[projlevel]).on(projlevel); size_t nbInter_ = 0, nbCells_ = 0; - intersect([&]( auto & interval, [[maybe_unused]] auto & index ) { - nbInter_ += 1; - nbCells_ += interval.size(); - }); - - // we get more interval / cells, than wanted because neighbour has bigger cells - if( nbInter_ > 0 && projlevel > level ){ - auto set_ = translate( currentMesh[ projlevel ], stencil ); - auto diff_ = difference( intersect, set_ ); - - nbInter_ = 0; - nbCells_ = 0; - diff_([&]( auto & interval, auto & index ) { + intersect( + [&](auto& interval, [[maybe_unused]] auto& index) + { nbInter_ += 1; nbCells_ += interval.size(); - // field( projlevel, i, index ) = world.rank() * 10; - interface[ nbi ][ projlevel ][ index ].add_interval( interval ); }); - }else{ - if( nbInter_ > 0 ){ - intersect([&]( auto & interval, auto & index ) { - interface[ nbi ][ projlevel ][ index ].add_interval( interval ); + // we get more interval / cells, than wanted because neighbour has bigger cells + if (nbInter_ > 0 && projlevel > level) + { + auto set_ = translate(currentMesh[projlevel], stencil); + auto diff_ = difference(intersect, set_); + + nbInter_ = 0; + nbCells_ = 0; + diff_( + [&](auto& interval, auto& index) + { + nbInter_ += 1; + nbCells_ += interval.size(); + // field( projlevel, i, index ) = world.rank() * 10; + interface[nbi][projlevel][index].add_interval(interval); }); + } + else + { + if (nbInter_ > 0) + { + intersect( + [&](auto& interval, auto& index) + { + interface[nbi][projlevel][index].add_interval(interval); + }); } } - } - } - } - } - std::vector interface_( n_neighbours ); - for(std::size_t nbi=0; nbi interface_(n_neighbours); + for (std::size_t nbi = 0; nbi < n_neighbours; ++nbi) + { + interface_[nbi] = {interface[nbi], true}; } return interface_; - } /** * Compute fluxes of cells between MPI processes. In -fake- MPI environment. To * use it in true MPI juste remove the loop over "irank", and replace irank by myrank; - * - */ - void compute_load_balancing_fluxes( std::vector & all ){ - - for( size_t irank = 0; irank < all.size(); ++irank ){ - - // number of cells + * + */ + void compute_load_balancing_fluxes(std::vector& all) + { + for (size_t irank = 0; irank < all.size(); ++irank) + { + // number of cells // supposing each cell has a cost of 1. ( no level dependency ) - int32_t load = all[ irank ]._load; + int32_t load = all[irank]._load; + + std::size_t n_neighbours = all[irank].neighbour.size(); - std::size_t n_neighbours = all[ irank ].neighbour.size(); - { std::cerr << "[compute_load_balancing_fluxes] Process # " << irank << " load : " << load << std::endl; std::cerr << "[compute_load_balancing_fluxes] Process # " << irank << " nneighbours : " << n_neighbours << std::endl; std::cerr << "[compute_load_balancing_fluxes] Process # " << irank << " neighbours : "; - for( size_t in=0; in loads; // data "load" to transfer to neighbour processes - all[ irank ].fluxes.resize( n_neighbours ); - std::fill( all[ irank ].fluxes.begin(), all[ irank ].fluxes.end(), 0); + all[irank].fluxes.resize(n_neighbours); + std::fill(all[irank].fluxes.begin(), all[irank].fluxes.end(), 0); const std::size_t n_iterations = 1; - for (std::size_t k = 0; k < n_iterations; ++k) { - + for (std::size_t k = 0; k < n_iterations; ++k) + { // numbers of neighboors processes for each neighbour process std::vector nb_neighbours; - if( irank == 0 ) std::cerr << "[compute_load_balancing_fluxes] Fluxes iteration # " << k << std::endl; - + if (irank == 0) + { + std::cerr << "[compute_load_balancing_fluxes] Fluxes iteration # " << k << std::endl; + } + // // get info from processes // mpi::all_gather(world, load, loads); // mpi::all_gather(world, m_mpi_neighbourhood.size(), nb_neighbours); @@ -672,29 +868,31 @@ namespace samurai{ int32_t load_np1 = load; // compute updated load for current process based on its neighbourhood - for (std::size_t j_rank = 0; j_rank < n_neighbours; ++j_rank){ - - auto neighbour_rank = static_cast( all[ irank ].neighbour[ j_rank ] ); - auto neighbour_load = all[ irank ].load[ j_rank ]; - auto diff_load = neighbour_load - load; + for (std::size_t j_rank = 0; j_rank < n_neighbours; ++j_rank) + { + auto neighbour_rank = static_cast(all[irank].neighbour[j_rank]); + auto neighbour_load = all[irank].load[j_rank]; + auto diff_load = neighbour_load - load; - std::size_t nb_neighbours_neighbour = all[ neighbour_rank ].neighbour.size(); + std::size_t nb_neighbours_neighbour = all[neighbour_rank].neighbour.size(); - double weight = 1. / static_cast( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); + double weight = 1. / static_cast(std::max(n_neighbours, nb_neighbours_neighbour) + 1); - int32_t transfertLoad = static_cast( std::lround( weight * static_cast( diff_load ) ) ); + int32_t transfertLoad = static_cast(std::lround(weight * static_cast(diff_load))); - all[ irank].fluxes[ j_rank ] += transfertLoad; + all[irank].fluxes[j_rank] += transfertLoad; load_np1 += transfertLoad; } // do check on load & fluxes ? - + { std::cerr << "fluxes : "; - for( size_t in=0; in - bool intersectionExists( Mesh_t & meshA, Mesh_t & meshB ){ - - using mesh_id_t = typename Mesh_t::mesh_id_t; + */ + template + bool intersectionExists(Mesh_t& meshA, Mesh_t& meshB) + { + using mesh_id_t = typename Mesh_t::mesh_id_t; // operation are on leaves cells only - auto meshA_ = meshA[ mesh_id_t::cells ]; - auto meshB_ = meshB[ mesh_id_t::cells ]; + auto meshA_ = meshA[mesh_id_t::cells]; + auto meshB_ = meshB[mesh_id_t::cells]; // get stencils for direction auto dirs = getDirection(); - for( size_t ist=0; ist( std::max( static_cast( meshA_.min_level() ), - static_cast( level ) - 1 ) ); - std::size_t maxlevel_check = std::min( meshA_.max_level(), level + static_cast( 1 ) ); - - for(std::size_t projlevel=minlevel_check; projlevel<=maxlevel_check; ++projlevel ){ + std::size_t minlevel_check = static_cast( + std::max(static_cast(meshA_.min_level()), static_cast(level) - 1)); + std::size_t maxlevel_check = std::min(meshA_.max_level(), level + static_cast(1)); + for (std::size_t projlevel = minlevel_check; projlevel <= maxlevel_check; ++projlevel) + { // translate meshB_ in "dir" direction - auto set = translate( meshB_[ level ], stencil ); - auto intersect = intersection( set, meshA_[ projlevel ] ).on( projlevel ); + auto set = translate(meshB_[level], stencil); + auto intersect = intersection(set, meshA_[projlevel]).on(projlevel); size_t nbInter_ = 0; - intersect([&]( [[maybe_unused]] auto & interval, [[maybe_unused]] auto & index ) { - nbInter_ += 1; - }); - - if( nbInter_ > 0 ) return true; + intersect( + [&]([[maybe_unused]] auto& interval, [[maybe_unused]] auto& index) + { + nbInter_ += 1; + }); + if (nbInter_ > 0) + { + return true; + } } - } - } return false; - } /** * Discover new neighbour connection that might arise during load balancing - */ + */ template - void discover_neighbour( Mesh_t & mesh ) { - + void discover_neighbour(Mesh_t& mesh) + { using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; boost::mpi::communicator world; - // DEBUG + // DEBUG std::ofstream logs; - logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs.open("log_" + std::to_string(world.rank()) + ".dat", std::ofstream::app); logs << "# discover_neighbour" << std::endl; // give access to geometricaly neighbour process rank and mesh - std::vector & neighbourhood = mesh.mpi_neighbourhood(); + std::vector& neighbourhood = mesh.mpi_neighbourhood(); // bool requireGeneralUpdate = false; - std::vector keepNeighbour( neighbourhood.size(), true ); + std::vector keepNeighbour(neighbourhood.size(), true); - std::vector> neighbourConnection( neighbourhood.size() ); + std::vector> neighbourConnection(neighbourhood.size()); // for each neighbour we check if there is an intersection with another one. - for( size_t nbi=0; nbi( mesh, neighbourhood[ nbi ].mesh ); + keepNeighbour[nbi] = intersectionExists(mesh, neighbourhood[nbi].mesh); // require update if a connection is lost // requireGeneralUpdate = requireGeneralUpdate || ( ! keepNeighbour[ nbi ] ); - if( ! keepNeighbour[ nbi ] ){ - logs << fmt::format("Loosing neighbour connection with {}", neighbourhood[ nbi ].rank) << std::endl; + if (!keepNeighbour[nbi]) + { + logs << fmt::format("Loosing neighbour connection with {}", neighbourhood[nbi].rank) << std::endl; } // check neighbour - neighbour connection - for( size_t nbj=nbi+1; nbj {}", neighbourhood[ nbi ].rank, - neighbourhood[ nbj ].rank ) << std::endl; - bool connected = intersectionExists( neighbourhood[ nbi ].mesh, neighbourhood[ nbj ].mesh ); - - if( connected ) { - neighbourConnection[ nbi ].emplace_back( neighbourhood[ nbj ].rank ); - neighbourConnection[ nbj ].emplace_back( neighbourhood[ nbi ].rank ); + for (size_t nbj = nbi + 1; nbj < neighbourhood.size(); ++nbj) + { + logs << fmt::format("Checking neighbourhood connection {} <-> {}", neighbourhood[nbi].rank, neighbourhood[nbj].rank) + << std::endl; + bool connected = intersectionExists(neighbourhood[nbi].mesh, neighbourhood[nbj].mesh); + + if (connected) + { + neighbourConnection[nbi].emplace_back(neighbourhood[nbj].rank); + neighbourConnection[nbj].emplace_back(neighbourhood[nbi].rank); } } } // communicate to neighbours the list of connection they have - for( size_t nbi=0; nbi Sending computed neighbour: {"; - for( const auto & i : neighbourConnection[ nbi ] ) + for (const auto& i : neighbourConnection[nbi]) + { logs << i << ", "; - logs << "} to process " << neighbourhood[ nbi ].rank << std::endl; - // end debug + } + logs << "} to process " << neighbourhood[nbi].rank << std::endl; + // end debug - world.send( neighbourhood[ nbi ].rank, 28, neighbourConnection[ nbi ] ); + world.send(neighbourhood[nbi].rank, 28, neighbourConnection[nbi]); } - - // map of current MPI neighbours processes rank + + // map of current MPI neighbours processes rank std::map _tmp; // key = rank, value = required - for( size_t nbi=0; nbi Receiving computed neighbour connection list from neighbour processes " << std::endl; - for( size_t nbi=0; nbi _rcv_neighbour_list; - world.recv( neighbourhood[ nbi ].rank, 28, _rcv_neighbour_list ); - - for(size_t in=0; in<_rcv_neighbour_list.size(); ++in ){ - if( _tmp.find( _rcv_neighbour_list[ in ] ) == _tmp.end() ){ - logs << "\t\t> New neighbour detected : " << _rcv_neighbour_list[ in ] << std::endl; + world.recv(neighbourhood[nbi].rank, 28, _rcv_neighbour_list); - _tmp[ _rcv_neighbour_list[ in ] ] = true; + for (size_t in = 0; in < _rcv_neighbour_list.size(); ++in) + { + if (_tmp.find(_rcv_neighbour_list[in]) == _tmp.end()) + { + logs << "\t\t> New neighbour detected : " << _rcv_neighbour_list[in] << std::endl; + + _tmp[_rcv_neighbour_list[in]] = true; // requireGeneralUpdate = requireGeneralUpdate || true; } } - } world.barrier(); - + // update neighbours ranks neighbourhood.clear(); - for( const auto & ni : _tmp ){ - neighbourhood.emplace_back( ni.first ); + for (const auto& ni : _tmp) + { + neighbourhood.emplace_back(ni.first); } - // debug + // debug logs << "New neighbourhood : {"; - for( size_t nbi=0; nbi // void perform_load_balancing_SFC( Mesh & mesh, int ndomains, Field_t & fake_mpi_rank ) { @@ -904,7 +1112,7 @@ namespace samurai{ // xt::xtensor_fixed> ij = { static_cast( tmp( 0 ) ), // static_cast( tmp( 1 ) ) }; - + // auto key = sfc.template getKey( ij ); // // std::cerr << "\t> Coord (" << ij( 0 ) << ", " << ij( 1 ) << ") ----> " << key << std::endl; @@ -917,10 +1125,9 @@ namespace samurai{ // // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; // std::cerr << "\t> Total number of cells : " << nbcells_tot << std::endl; -// std::cout << "\t> Perfect load-balancing (weight-cell = 1.) : " << static_cast( nbcells_tot / ndomains ) +// std::cout << "\t> Perfect load-balancing (weight-cell = 1.) : " << static_cast( nbcells_tot / ndomains ) // << " / MPI" << std::endl; - // size_t cindex = 0; // for(const auto & item : sfc_map ) { // auto sfc_key = item.first; @@ -960,7 +1167,7 @@ namespace samurai{ // // get Logical coordinate or first cell // xt::xtensor_fixed> icell; - + // icp == Interval_CellPosition::LAST ? icell( 0 ) = inter.end - 1 : icell( 0 ) = inter.start; // for(int idim=0; idim> ijk; - -// if constexpr ( dim == 2 ) ijk = { static_cast( icell( 0 ) ), + +// if constexpr ( dim == 2 ) ijk = { static_cast( icell( 0 ) ), // static_cast( icell( 1 ) ) }; - -// if constexpr ( dim == 3 ) ijk = { static_cast( icell( 0 ) ), + +// if constexpr ( dim == 3 ) ijk = { static_cast( icell( 0 ) ), // static_cast( icell( 1 ) ), // static_cast( icell( 2 ) ) }; - + // sfc_map[ sfc.getKey( ijk ) ] = { level, inter, index }; // ninterval ++; @@ -990,10 +1197,9 @@ namespace samurai{ // // std::cerr << "\n\t> Morton index [" << min << ", " << max << "]" << std::endl; // std::cerr << "\t> Total number of interval : " << ninterval << std::endl; -// std::cout << "\t> Perfect load-balancing (weight-interval = 1.) : " << static_cast( ninterPerProc ) +// std::cout << "\t> Perfect load-balancing (weight-interval = 1.) : " << static_cast( ninterPerProc ) // << " / MPI" << std::endl; - // size_t cindex = 0; // for(const auto & item : sfc_map ) { // int fake_rank = std::floor( cindex / ninterPerProc ); @@ -1007,13 +1213,13 @@ namespace samurai{ // } // /** -// * +// * // * Global contains the global mesh. Since this is a toy function, it contains the union of all mesh. // * meshes contains the mesh of each MPI process ( or let say the mesh of neighbour processes ...) -// * +// * // */ // template -// void perform_load_balancing_diffusion( AMesh_t & global, std::vector & meshes, int ndomains, +// void perform_load_balancing_diffusion( AMesh_t & global, std::vector & meshes, int ndomains, // const std::vector & all, Field_t & fake_mpi_rank ) { // using CellList_t = typename AMesh_t::cl_type; @@ -1029,7 +1235,7 @@ namespace samurai{ // double wght_tot = 0.; // samurai::for_each_cell( meshes[ m_ ], [&]( const auto & cell ) { - + // // [OPTIMIZATION] precompute weight as array // double wght = 1. / ( 1 << ( maxlevel - cell.level ) ); @@ -1057,7 +1263,7 @@ namespace samurai{ // std::vector new_meshes( ndomains ); // std::vector exchanged( ndomains ); -// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains // std::cerr << "\t> Working on domains # " << m_ << std::endl; @@ -1065,7 +1271,7 @@ namespace samurai{ // // id des neighbours dans le tableau de la structure MPI_Load_Balance // // attention différent du rank mpi ! -// std::vector id_send; +// std::vector id_send; // int n_neighbours = static_cast( all[ m_ ].neighbour.size() ); // for(std::size_t nbi = 0; nbi( tmp.coord( idim ) / 0.5 ); @@ -1107,12 +1313,12 @@ namespace samurai{ // } // std::cerr << "\t\t> Number RECV : " << id_send.size() << std::endl; - + // std::vector already_given( n_neighbours, 0 ); // for_each_interval( meshes[ m_ ], [&]( std::size_t level, const auto& interval, const auto& index ){ // Coord_t ibar; - + // std::cerr << "\t\t> Interval [" << interval.start << ", " << interval.end << "[" << std::endl; // double dm = 1. / (1 << level ); @@ -1141,9 +1347,9 @@ namespace samurai{ // std::cerr << "\t\t\t> NbCells of this interval " << interval.size() << std::endl; // std::cerr << "\t\t\t> fluxes for this neighbour : " << all[ m_ ].fluxes[ id_send[ nbi ] ] << std::endl; -// if( dist < winner_dist && +// if( dist < winner_dist && // already_given[ neighbour_rank ] + interval.size() <= (- all[ m_ ].fluxes[ id_send[ nbi ] ]) ){ - + // winner_id = id_send[ nbi ]; // winner_dist = dist; // } @@ -1161,12 +1367,11 @@ namespace samurai{ // }); - // meshes[ m_ ] = { new_meshes[ m_ ], true }; // } -// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains // for( int level=global.min_level(); level<=global.max_level(); ++level ) { // auto intersect = intersection( global[ level ], meshes[ m_ ][ level ]); @@ -1188,7 +1393,8 @@ namespace samurai{ // } // template -// void perform_load_balancing_diffusion( AMesh_t & global, int ndomains, const std::vector & all, Field_t & fake_mpi_rank ) { +// void perform_load_balancing_diffusion( AMesh_t & global, int ndomains, const std::vector & all, Field_t & +// fake_mpi_rank ) { // using CellList_t = typename AMesh_t::cl_type; // using cell_t = typename AMesh_t::cell_t; @@ -1205,7 +1411,7 @@ namespace samurai{ // double wght_tot = 0.; // samurai::for_each_cell( meshes[ m_ ], [&]( const auto & cell ) { - + // // [OPTIMIZATION] precompute weight as array // double wght = 1. / ( 1 << ( maxlevel - cell.level ) ); @@ -1233,15 +1439,15 @@ namespace samurai{ // std::vector new_meshes( ndomains ); // std::vector exchanged( ndomains ); -// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains // std::cerr << "\t> Working on domains # " << m_ << std::endl; // auto dist = samurai::make_field( "rank", meshes[ m_ ] ); // // neighbours offset to which data must be sent. -// // This is not equivalent to the MPI rank -// std::vector id_send; +// // This is not equivalent to the MPI rank +// std::vector id_send; // int n_neighbours = static_cast( all[ m_ ].neighbour.size() ); // for(std::size_t nbi = 0; nbi already_given( n_neighbours, 0 ); -// // loop over each interval of the current ( or meshes[ m_ ] ) domain and check if it needs to be sent +// // loop over each interval of the current ( or meshes[ m_ ] ) domain and check if it needs to be sent // // to a given neighbour // for_each_interval( meshes[ m_ ], [&]( std::size_t level, const auto& interval, const auto& index ){ // Coord_t ibar; - + // std::cerr << "\t\t> Interval [" << interval.start << ", " << interval.end << "[" << std::endl; // double dm = 1. / (1 << level ); @@ -1289,9 +1495,9 @@ namespace samurai{ // std::cerr << "\t\t\t> NbCells of this interval " << interval.size() << std::endl; // std::cerr << "\t\t\t> fluxes for this neighbour : " << all[ m_ ].fluxes[ id_send[ nbi ] ] << std::endl; -// if( dist < winner_dist && +// if( dist < winner_dist && // already_given[ neighbour_rank ] + interval.size() <= (- all[ m_ ].fluxes[ id_send[ nbi ] ]) ){ - + // winner_id = id_send[ nbi ]; // winner_dist = dist; // } @@ -1309,12 +1515,11 @@ namespace samurai{ // }); - // meshes[ m_ ] = { new_meshes[ m_ ], true }; // } -// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains +// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains // for( int level=global.min_level(); level<=global.max_level(); ++level ) { // auto intersect = intersection( global[ level ], meshes[ m_ ][ level ]); From 7af599b8fba9da5391221bdf6f8b89019bcc4e09 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 18 Apr 2024 17:05:37 +0200 Subject: [PATCH 052/170] clang format --- include/samurai/load_balancing_sfc.hpp | 423 +++++++++++++------------ 1 file changed, 226 insertions(+), 197 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index abc4c9955..bc5ffd30d 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -4,235 +4,264 @@ #include "load_balancing.hpp" -template -class SFC_LoadBalancer_interval : public samurai::LoadBalancer> { +template +class SFC_LoadBalancer_interval : public samurai::LoadBalancer> +{ + private: - private: - SFC_type_t _sfc; - int _ndomains; - int _rank; + SFC_type_t _sfc; + int _ndomains; + int _rank; - const double TRANSFER_PERCENT = 0.42; + const double TRANSFER_PERCENT = 0.42; - public: - - SFC_LoadBalancer_interval() { + public: + SFC_LoadBalancer_interval() + { #ifdef SAMURAI_WITH_MPI - boost::mpi::communicator world; - _ndomains = world.size(); - _rank = world.rank(); + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); #else - _ndomains = 1; - _rank = 0; + _ndomains = 1; + _rank = 0; #endif + } + + inline std::string getName() const + { + return "SFC_" + _sfc.getName() + "_LB"; + } + + template + Mesh load_balance_impl(Mesh& mesh, Fields&... data) + { + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using inter_t = samurai::Interval; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = samurai::CellArray; + + struct Data_t + { + size_t level; + inter_t interval; + xt::xtensor_fixed> indices; + bool given; + }; + + boost::mpi::communicator world; + + // For debug + std::ofstream logs; + logs.open("log_" + std::to_string(_rank) + ".dat", std::ofstream::app); + logs << "# New load balancing (load_balancing_sfc)" << std::endl; + + // SFC key (used for implicit sorting through map mechanism) + std::map sfc_map; + + size_t ninterval = 0; + samurai::for_each_interval(mesh, + [&](std::size_t level, const auto& inter, const auto& index) + { + // get Logical coordinate or first cell + xt::xtensor_fixed> icell; + + // first element of interval + icell(0) = inter.start; + for (int idim = 0; idim < dim - 1; ++idim) + { + icell(idim + 1) = index(idim); + } + + // convert logical coordinate to max level logical coordinates + for (int idim = 0; idim < dim; ++idim) + { + icell(idim) = icell(idim) << (mesh.max_level() - level + 1); + } + + // this is where think can get nasty, we expect indices to be positive values !! + xt::xtensor_fixed> ijk; + for (size_t idim = 0; idim < dim; ++idim) + { + ijk(idim) = static_cast(icell(idim)); + } + + sfc_map[_sfc.template getKey(ijk)] = {level, inter, index, false}; + + ninterval++; + }); + + assert(ninterval == sfc_map.size()); + + // Key boundaries of current process - unused for now + // SFC_key_t interval[ 2 ] = { sfc_map.begin()->first, sfc_map.rbegin()->first }; + // logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; + + std::vector load_interval; + int my_load_i = static_cast(samurai::cmptLoad(mesh)); + boost::mpi::all_gather(world, my_load_i, load_interval); + + // compute load to transfer to neighbour rank-1, rank+1 + int neighbour_rank_prev = -1, neighbour_rank_next = -1; + int transfer_load_prev = 0, transfer_load_next = 0; + + // define neighbour processes for load-balancing, not geometrical neighbour ! + if (_rank > 0) + { + neighbour_rank_prev = _rank - 1; + // transfer TRANSFER_PERCENT % max of difference + transfer_load_prev = -static_cast((my_load_i - load_interval[static_cast(neighbour_rank_prev)]) + * TRANSFER_PERCENT); } - inline std::string getName() const { return "SFC_" + _sfc.getName() + "_LB"; } - - template - Mesh load_balance_impl( Mesh & mesh, Fields&... data ){ - - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; - using inter_t = samurai::Interval; - using CellList_t = typename Mesh_t::cl_type; - using CellArray_t = samurai::CellArray; - - struct Data_t { - size_t level; - inter_t interval; - xt::xtensor_fixed> indices; - bool given; - }; - - boost::mpi::communicator world; - - // For debug - std::ofstream logs; - logs.open( "log_" + std::to_string( _rank ) + ".dat", std::ofstream::app ); - logs << "# New load balancing (load_balancing_sfc)" << std::endl; - - // SFC key (used for implicit sorting through map mechanism) - std::map sfc_map; - - size_t ninterval = 0; - samurai::for_each_interval( mesh, [&]( std::size_t level, const auto& inter, const auto& index ){ - - // get Logical coordinate or first cell - xt::xtensor_fixed> icell; - - // first element of interval - icell( 0 ) = inter.start; - for(int idim=0; idim((my_load_i - load_interval[static_cast(neighbour_rank_next)]) + * TRANSFER_PERCENT); + } - // this is where think can get nasty, we expect indices to be positive values !! - xt::xtensor_fixed> ijk; - for(size_t idim=0; idim( icell( idim ) ); + logs << "Neighbour prev : " << neighbour_rank_prev << ", transfer of loads : " << transfer_load_prev << std::endl; + logs << "Neighbour next : " << neighbour_rank_next << ", transfer of loads : " << transfer_load_next << std::endl; + + /** + * In this section we update the current process mesh data based on what is sent and receive + * + */ + + CellList_t new_cl; // this will contains the final mesh of the current process + + // need send data to prev neighbour + if (neighbour_rank_prev >= 0 && transfer_load_prev != 0) + { + if (transfer_load_prev < 0) + { + CellList_t cl_to_send; + + // give n-smallest morton keys to prev neighbour + size_t niter_send = 0; + for (auto iter = sfc_map.begin(); iter != sfc_map.end(); ++iter) + { + if (transfer_load_prev < 0 && my_load_i > 0) + { + cl_to_send[iter->second.level][iter->second.indices].add_interval(iter->second.interval); + iter->second.given = true; // flag "interval" has been sent + my_load_i -= 1; + transfer_load_prev += 1; + niter_send++; + } + else + { + break; + } } - - sfc_map[ _sfc.template getKey( ijk ) ] = { level, inter, index, false }; - ninterval ++; - }); + CellArray_t ca_to_send = {cl_to_send, false}; - assert( ninterval == sfc_map.size() ); + logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_prev << std::endl; - // Key boundaries of current process - unused for now - // SFC_key_t interval[ 2 ] = { sfc_map.begin()->first, sfc_map.rbegin()->first }; - // logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; + world.send(neighbour_rank_prev, 42, ca_to_send); - std::vector load_interval; - int my_load_i = static_cast( samurai::cmptLoad( mesh ) ); - boost::mpi::all_gather( world, my_load_i, load_interval ); + // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); + // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; - // compute load to transfer to neighbour rank-1, rank+1 - int neighbour_rank_prev = -1, neighbour_rank_next = -1; - int transfer_load_prev = 0, transfer_load_next = 0; - - // define neighbour processes for load-balancing, not geometrical neighbour ! - if( _rank > 0 ) { - neighbour_rank_prev = _rank - 1; - // transfer TRANSFER_PERCENT % max of difference - transfer_load_prev = - static_cast( ( my_load_i - load_interval[ static_cast( neighbour_rank_prev ) ] ) * TRANSFER_PERCENT ); + // mesh.remove( ca_to_send ); } - - if( _rank < _ndomains - 1 ) { - neighbour_rank_next = _rank + 1; - // transfer TRANSFER_PERCENT % max of difference - transfer_load_next = - static_cast( ( my_load_i - load_interval[ static_cast( neighbour_rank_next ) ] ) * TRANSFER_PERCENT ); + else + { + // need recv + CellArray_t ca_to_rcv; + world.recv(neighbour_rank_prev, 42, ca_to_rcv); + + logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_prev << std::endl; + + // mesh.merge( ca_to_rcv ); + // add to CL what we just receive + samurai::for_each_interval(ca_to_rcv, + [&](std::size_t level, const auto& interval, const auto& index) + { + new_cl[level][index].add_interval(interval); + }); } + } - logs << "Neighbour prev : " << neighbour_rank_prev << ", transfer of loads : " << transfer_load_prev << std::endl; - logs << "Neighbour next : " << neighbour_rank_next << ", transfer of loads : " << transfer_load_next << std::endl; - - /** - * In this section we update the current process mesh data based on what is sent and receive - * - */ - - CellList_t new_cl; // this will contains the final mesh of the current process - - // need send data to prev neighbour - if( neighbour_rank_prev >= 0 && transfer_load_prev != 0 ){ - - if( transfer_load_prev < 0 ){ - CellList_t cl_to_send; - - // give n-smallest morton keys to prev neighbour - size_t niter_send = 0; - for (auto iter = sfc_map.begin(); iter != sfc_map.end(); ++iter) { - if( transfer_load_prev < 0 && my_load_i > 0 ){ - cl_to_send[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); - iter->second.given = true; // flag for "interval" has been sent - my_load_i -= 1; - transfer_load_prev += 1; - niter_send ++; - }else{ - break; - } + if (neighbour_rank_next > 0 && transfer_load_next != 0) + { + if (transfer_load_next < 0) + { + CellList_t cl_to_send; + + // give n-smallest morton keys to prev neighbour + size_t niter_send = 0; + for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) + { + if (transfer_load_next < 0 && my_load_i > 0) + { + cl_to_send[iter->second.level][iter->second.indices].add_interval(iter->second.interval); + iter->second.given = true; + my_load_i -= 1; + transfer_load_next += 1; + niter_send++; } - - CellArray_t ca_to_send = { cl_to_send, false }; - - logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_prev << std::endl; - - world.send( neighbour_rank_prev, 42, ca_to_send ); - - // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); - // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; - - // mesh.remove( ca_to_send ); - - }else{ - // need recv - CellArray_t ca_to_rcv; - world.recv( neighbour_rank_prev, 42, ca_to_rcv ); - - logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_prev << std::endl; - - // mesh.merge( ca_to_rcv ); - // add to CL what we just receive - samurai::for_each_interval( ca_to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ - new_cl[ level ][ index ].add_interval( interval ); - }); - - } - - } - - if( neighbour_rank_next > 0 && transfer_load_next != 0 ){ - - if( transfer_load_next < 0 ){ - CellList_t cl_to_send; - - // give n-smallest morton keys to prev neighbour - size_t niter_send = 0; - for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) { - if( transfer_load_next < 0 && my_load_i > 0 ){ - cl_to_send[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); - iter->second.given = true; - my_load_i -= 1; - transfer_load_next += 1; - niter_send ++; - }else{ - break; - } + else + { + break; } + } - CellArray_t ca_to_send = { cl_to_send, false }; - - logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_next << std::endl; - - world.send( neighbour_rank_next, 42, ca_to_send ); - - // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); - // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; - - mesh.remove( ca_to_send ); - - }else{ - // need recv - CellArray_t ca_to_rcv; - world.recv( neighbour_rank_next, 42, ca_to_rcv ); + CellArray_t ca_to_send = {cl_to_send, false}; - logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_next << std::endl; + logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_next << std::endl; - // mesh.merge( ca_to_rcv ); - // add to CL what we just receive - samurai::for_each_interval( ca_to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ - new_cl[ level ][ index ].add_interval( interval ); - }); + world.send(neighbour_rank_next, 42, ca_to_send); - } + // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); + // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; + // mesh.remove( ca_to_send ); } - - // last loop over the map to add what is left - for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) { - if( ! iter->second.given ){ - new_cl[ iter->second.level][ iter->second.indices ].add_interval( iter->second.interval ); - } + else + { + // need recv + CellArray_t ca_to_rcv; + world.recv(neighbour_rank_next, 42, ca_to_rcv); + + logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_next << std::endl; + + // mesh.merge( ca_to_rcv ); + // add to CL what we just receive + samurai::for_each_interval(ca_to_rcv, + [&](std::size_t level, const auto& interval, const auto& index) + { + new_cl[level][index].add_interval(interval); + }); } + } - Mesh_t new_mesh( new_cl, mesh.min_level(), mesh.max_level(), mesh.mpi_neighbourhood() ); - // mesh.update_mesh_neighbour(); done already in new_mesh constructor + // last loop over the map to add what is left + for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) + { + if (!iter->second.given) + { + new_cl[iter->second.level][iter->second.indices].add_interval(iter->second.interval); + } + } - // update neighbour connectivity - samurai::discover_neighbour( new_mesh ); - samurai::discover_neighbour( new_mesh ); + Mesh_t new_mesh(new_cl, mesh); + logs << "\t> Number of cells in OLD mesh(mesh_id_t::cells) : " << mesh.nb_cells(Mesh_t::mesh_id_t::cells) << std::endl; + logs << "\t> Number of cells in NEW mesh(mesh_id_t::cells) : " << new_mesh.nb_cells(Mesh_t::mesh_id_t::cells) << std::endl; - return new_mesh; + logs << "\t> Number of cells in OLD mesh(mesh_id_t::all) : " << mesh.nb_cells() << std::endl; + logs << "\t> Number of cells in NEW mesh(mesh_id_t::all) : " << new_mesh.nb_cells() << std::endl; - } + // update neighbour connectivity + // samurai::discover_neighbour(new_mesh); + // samurai::discover_neighbour(new_mesh); + return new_mesh; + } }; \ No newline at end of file From 86c8089ea3ea39945a7e61618f00fd57f0fad16e Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 18 Apr 2024 17:05:52 +0200 Subject: [PATCH 053/170] update advection with dynamic load balancing --- demos/FiniteVolume/advection_2d.cpp | 51 +++++++++++++++-------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index f97af6deb..9e2dd3384 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -24,7 +24,7 @@ namespace fs = std::filesystem; template -auto init(Mesh& mesh, const double radius, const double x_center, const double y_center ) +auto init(Mesh& mesh, const double radius, const double x_center, const double y_center) { auto u = samurai::make_field("u", mesh); @@ -32,7 +32,7 @@ auto init(Mesh& mesh, const double radius, const double x_center, const double y mesh, [&](auto& cell) { - auto center = cell.center(); + auto center = cell.center(); // const double radius = .2; // const double x_center = 0.3; // const double y_center = 0.3; @@ -165,8 +165,8 @@ void save(const fs::path& path, const std::string& filename, const Field& u, con samurai::for_each_cell(mesh, [&](const auto& cell) { - level_[cell] = cell.level; - domain_[ cell ] = mrank; + level_[cell] = cell.level; + domain_[cell] = mrank; }); #ifdef SAMURAI_WITH_MPI mpi::communicator world; @@ -239,14 +239,14 @@ int main(int argc, char* argv[]) const double dt_save = Tf / static_cast(nfiles); double t = 0.; - auto u = init( mesh, radius, x_center, y_center ); + auto u = init(mesh, radius, x_center, y_center); samurai::make_bc>(u, 0.); auto unp1 = samurai::make_field("unp1", mesh); - myTimers.start( "make_MRAdapt_init" ); + myTimers.start("make_MRAdapt_init"); auto MRadaptation = samurai::make_MRAdapt(u); MRadaptation(mr_epsilon, mr_regularity); - myTimers.stop( "make_MRAdapt_init" ); + myTimers.stop("make_MRAdapt_init"); save(path, filename, u, "_init"); @@ -257,20 +257,23 @@ int main(int argc, char* argv[]) while (t != Tf) { - - if( nt % 20 == 0 && nt > 1 ){ - std::cout << "\t> Load balancing mesh ... " << std::endl; + if (nt % 20 == 0 && nt > 1) + { + std::cout << "\t> Load balancing mesh ... " << std::endl; myTimers.start("load-balancing"); - auto new_mesh = balancer.load_balance( mesh ); + balancer.load_balance(mesh, u); myTimers.stop("load-balancing"); - - unp1.resize(); - u.resize(); + + // const std::string suffix = fmt::format("_loadbalanced_{}", nt); + // save(path, filename, u, suffix); + + // samurai::finalize(); + // return 0; } - myTimers.start( "MRadaptation" ); + myTimers.start("MRadaptation"); MRadaptation(mr_epsilon, mr_regularity); - myTimers.stop( "MRadaptation" ); + myTimers.stop("MRadaptation"); t += dt; if (t > Tf) @@ -281,15 +284,15 @@ int main(int argc, char* argv[]) std::cout << fmt::format("iteration {}: t = {}, dt = {}", nt++, t, dt) << std::endl; - myTimers.start( "update_ghost_mr" ); + myTimers.start("update_ghost_mr"); samurai::update_ghost_mr(u); - myTimers.stop( "update_ghost_mr" ); + myTimers.stop("update_ghost_mr"); unp1.resize(); - myTimers.start( "upwind" ); + myTimers.start("upwind"); unp1 = u - dt * samurai::upwind(a, u); - myTimers.stop( "upwind" ); + myTimers.stop("upwind"); if (correction) { @@ -298,15 +301,15 @@ int main(int argc, char* argv[]) std::swap(u.array(), unp1.array()); - myTimers.start( "I/O" ); + myTimers.start("I/O"); // if (t >= static_cast(nsave + 1) * dt_save || t == Tf || true ) - if( nt % 20 == 0 ) { + if (nt % 20 == 0) + { // const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; const std::string suffix = fmt::format("_ite_{}", nt); save(path, filename, u, suffix); } - myTimers.stop( "I/O" ); - + myTimers.stop("I/O"); } myTimers.print(); From f9a7953951077176a0a8334f2a2b50453e11c613 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 23 Apr 2024 15:01:48 +0200 Subject: [PATCH 054/170] clang format --- tests/test_mrmesh.cpp | 87 ++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 46 deletions(-) diff --git a/tests/test_mrmesh.cpp b/tests/test_mrmesh.cpp index a5ec3f531..10be6758c 100644 --- a/tests/test_mrmesh.cpp +++ b/tests/test_mrmesh.cpp @@ -2,9 +2,9 @@ #include #include -#include -#include #include +#include +#include #include @@ -12,10 +12,10 @@ namespace samurai { /* - * test MR Mesh; - */ - TEST(mrmesh, test_nbcells2D){ - + * test MR Mesh; + */ + TEST(mrmesh, test_nbcells2D) + { constexpr int dim = 2; using Config = samurai::MRConfig; @@ -42,27 +42,26 @@ namespace samurai cl[2][{14}].add_interval({14, 16}); cl[2][{15}].add_interval({14, 16}); - Mesh_t mesh( cl, 0, 2 ); + Mesh_t mesh(cl, 0, 2); - ASSERT_EQ( mesh.min_level(), 0 ); - ASSERT_EQ( mesh.max_level(), 2 ); + ASSERT_EQ(mesh.min_level(), 0); + ASSERT_EQ(mesh.max_level(), 2); - std::vector nCellPerLevel_withGhost = { 36, 48, 32 }; // including ghost - for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ - ASSERT_EQ( mesh.nb_cells( mesh.min_level() ), - nCellPerLevel_withGhost[ mesh.min_level() ] ); + std::vector nCellPerLevel_withGhost = {36, 48, 32}; // including ghost + for (size_t ilvl = mesh.min_level(); ilvl <= mesh.max_level(); ++ilvl) + { + ASSERT_EQ(mesh.nb_cells(mesh.min_level()), nCellPerLevel_withGhost[mesh.min_level()]); } - std::vector nCellPerLevel_leaves = { 11, 18, 8 }; // not including ghost - for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ - ASSERT_EQ( mesh.nb_cells( mesh.min_level(), samurai::MRMeshId::cells ), - nCellPerLevel_leaves[ mesh.min_level() ] ); + std::vector nCellPerLevel_leaves = {11, 18, 8}; // not including ghost + for (size_t ilvl = mesh.min_level(); ilvl <= mesh.max_level(); ++ilvl) + { + ASSERT_EQ(mesh.nb_cells(mesh.min_level(), samurai::MRMeshId::cells), nCellPerLevel_leaves[mesh.min_level()]); } - } - TEST(mrmesh, test_exist2D){ - + TEST(mrmesh, test_exist2D) + { constexpr int dim = 2; using Config = samurai::MRConfig; @@ -89,19 +88,19 @@ namespace samurai cl[2][{14}].add_interval({14, 16}); cl[2][{15}].add_interval({14, 16}); - Mesh_t mesh( cl, 0, 2 ); + Mesh_t mesh(cl, 0, 2); Interval i{0, 3, 0}; - + // mesh.exists( samurai::MRMeshId::cells, 1, ) } - TEST(mrmesh, test_merge){ - + TEST(mrmesh, test_merge) + { constexpr int dim = 2; - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; using CellArray_t = Mesh_t::ca_type; samurai::CellList cl; @@ -110,27 +109,26 @@ namespace samurai cl[0][{1}].add_interval({3, 4}); cl[1][{2}].add_interval({2, 6}); - Mesh_t mesh( cl, 0, 2 ); - ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 10 ); + Mesh_t mesh(cl, 0, 2); + ASSERT_EQ(mesh.nb_cells(Mesh_t::mesh_id_t::cells), 10); samurai::CellList to_add_cl; to_add_cl[1][{3}].add_interval({2, 6}); - CellArray_t to_add_ca = { to_add_cl, false }; + CellArray_t to_add_ca = {to_add_cl, false}; - mesh.merge( to_add_ca ); + mesh.merge(to_add_ca); // technically not sufficient to be sure the merge was done properly - ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 14 ); - + ASSERT_EQ(mesh.nb_cells(Mesh_t::mesh_id_t::cells), 14); } - TEST(mrmesh, test_remove){ - + TEST(mrmesh, test_remove) + { constexpr int dim = 2; - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; using CellArray_t = Mesh_t::ca_type; samurai::CellList cl; @@ -140,22 +138,19 @@ namespace samurai cl[1][{2}].add_interval({2, 6}); cl[1][{3}].add_interval({2, 6}); - Mesh_t mesh( cl, 0, 2 ); - ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 14 ); + Mesh_t mesh(cl, 0, 2); + ASSERT_EQ(mesh.nb_cells(Mesh_t::mesh_id_t::cells), 14); samurai::CellList to_rm_cl; - to_rm_cl[0][{1}].add_interval({3,4}); + to_rm_cl[0][{1}].add_interval({3, 4}); to_rm_cl[1][{3}].add_interval({2, 6}); - - CellArray_t to_rm_ca = { to_rm_cl, false }; - mesh.remove( to_rm_ca ); + CellArray_t to_rm_ca = {to_rm_cl, false}; - // technically not sufficient to be sure the remove was done properly - ASSERT_EQ( mesh.nb_cells( Mesh_t::mesh_id_t::cells ), 9 ); + mesh.remove(to_rm_ca); + // technically not sufficient to be sure the remove was done properly + ASSERT_EQ(mesh.nb_cells(Mesh_t::mesh_id_t::cells), 9); } - - } \ No newline at end of file From f9a4dcaf38dd01c30243d01b39c32f16e21d92aa Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 23 Apr 2024 15:02:08 +0200 Subject: [PATCH 055/170] fix mpi for unit tests --- tests/main.cpp | 16 +++++++++------- tests/test_adapt.cpp | 5 +++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/main.cpp b/tests/main.cpp index a3344fda4..9767eb035 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -1,13 +1,15 @@ -#ifdef SAMURAI_WITH_MPI -#include -#endif #include +#include int main(int argc, char* argv[]) { -#ifdef SAMURAI_WITH_MPI - boost::mpi::environment env(argc, argv); -#endif + samurai::initialize(); + ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + + int ret = RUN_ALL_TESTS(); + + samurai::finalize(); + + return ret; } diff --git a/tests/test_adapt.cpp b/tests/test_adapt.cpp index 5ba4637e1..c56e1df2a 100644 --- a/tests/test_adapt.cpp +++ b/tests/test_adapt.cpp @@ -27,7 +27,7 @@ namespace samurai TYPED_TEST(adapt_test, mutliple_fields) { - ::samurai::initialize(); + // ::samurai::initialize(); static constexpr std::size_t dim = TypeParam::value; using config = MRConfig; @@ -38,6 +38,7 @@ namespace samurai auto adapt = make_MRAdapt(u_1, u_2, u_3); adapt(1e-4, 2); - ::samurai::finalize(); + + // ::samurai::finalize(); } } From 0d231e3ae6d520b8f6da913f4d11babe55bf33fd Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 23 Apr 2024 15:02:18 +0200 Subject: [PATCH 056/170] clang format --- tests/test_sfc.cpp | 170 +++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 84 deletions(-) diff --git a/tests/test_sfc.cpp b/tests/test_sfc.cpp index 497993924..e72ce4bb4 100644 --- a/tests/test_sfc.cpp +++ b/tests/test_sfc.cpp @@ -2,8 +2,8 @@ #include #include -#include #include +#include #include @@ -13,136 +13,138 @@ namespace samurai using Coord_3D_t = xt::xtensor_fixed>; /* - * test computation of morton indexes 2D - */ - TEST(sfc, morton2D_getKey){ + * test computation of morton indexes 2D + */ + TEST(sfc, morton2D_getKey) + { constexpr int dim = 2; SFC morton; - Coord_2D_t ij = { 0, 0 }; - auto key1 = morton.getKey( ij ); - ASSERT_EQ( key1, 0 ); + Coord_2D_t ij = {0, 0}; + auto key1 = morton.getKey(ij); + ASSERT_EQ(key1, 0); - ij = { 2, 1 }; - auto key2 = morton.getKey( ij ); - ASSERT_EQ( key2, 6 ); + ij = {2, 1}; + auto key2 = morton.getKey(ij); + ASSERT_EQ(key2, 6); - ij = { 5, 6 }; - auto key3 = morton.getKey( ij ); - ASSERT_EQ( key3, 57 ); + ij = {5, 6}; + auto key3 = morton.getKey(ij); + ASSERT_EQ(key3, 57); - ij = { 0, 4 }; - auto key4 = morton.getKey( ij ); - ASSERT_EQ( key4, 32 ); + ij = {0, 4}; + auto key4 = morton.getKey(ij); + ASSERT_EQ(key4, 32); - ij = { 4, 0 }; - auto key5 = morton.getKey( ij ); - ASSERT_EQ( key5, 16 ); + ij = {4, 0}; + auto key5 = morton.getKey(ij); + ASSERT_EQ(key5, 16); } /* - * test computation of (i,j) logical coordinate from morton indexes 2D - */ - TEST(sfc, morton2D_getCoordinates){ + * test computation of (i,j) logical coordinate from morton indexes 2D + */ + TEST(sfc, morton2D_getCoordinates) + { constexpr int dim = 2; SFC morton; - auto ij = morton.getCoordinates( static_cast( 0 ) ); - ASSERT_EQ( ij(0), 0 ); - ASSERT_EQ( ij(1), 0 ); + auto ij = morton.getCoordinates(static_cast(0)); + ASSERT_EQ(ij(0), 0); + ASSERT_EQ(ij(1), 0); - auto ij2 = morton.getCoordinates( static_cast( 31 ) ); - ASSERT_EQ( ij2(0), 7 ); - ASSERT_EQ( ij2(1), 3 ); + auto ij2 = morton.getCoordinates(static_cast(31)); + ASSERT_EQ(ij2(0), 7); + ASSERT_EQ(ij2(1), 3); - auto ij3 = morton.getCoordinates( static_cast( 51 ) ); - ASSERT_EQ( ij3(0), 5 ); - ASSERT_EQ( ij3(1), 5 ); + auto ij3 = morton.getCoordinates(static_cast(51)); + ASSERT_EQ(ij3(0), 5); + ASSERT_EQ(ij3(1), 5); - auto ij4 = morton.getCoordinates( static_cast( 39 ) ); - ASSERT_EQ( ij4(0), 3 ); - ASSERT_EQ( ij4(1), 5 ); + auto ij4 = morton.getCoordinates(static_cast(39)); + ASSERT_EQ(ij4(0), 3); + ASSERT_EQ(ij4(1), 5); } /* - * test computation of morton indexes 3D - */ - TEST(sfc, morton3D_getKey){ + * test computation of morton indexes 3D + */ + TEST(sfc, morton3D_getKey) + { constexpr int dim = 3; - - SFC morton; - Coord_3D_t ijk = { 0, 0, 0 }; - auto key1 = morton.getKey( ijk ); - ASSERT_EQ( key1, 0 ); + SFC morton; - ijk = { 1, 1, 0 }; - auto key2 = morton.getKey( ijk ); - ASSERT_EQ( key2, 3 ); + Coord_3D_t ijk = {0, 0, 0}; + auto key1 = morton.getKey(ijk); + ASSERT_EQ(key1, 0); - ijk = { 0, 0, 1 }; - auto key3 = morton.getKey( ijk ); - ASSERT_EQ( key3, 4 ); + ijk = {1, 1, 0}; + auto key2 = morton.getKey(ijk); + ASSERT_EQ(key2, 3); - ijk = { 5, 9, 1 }; - auto key4 = morton.getKey( ijk ); - ASSERT_EQ( key4, 1095 ); + ijk = {0, 0, 1}; + auto key3 = morton.getKey(ijk); + ASSERT_EQ(key3, 4); + ijk = {5, 9, 1}; + auto key4 = morton.getKey(ijk); + ASSERT_EQ(key4, 1095); } /* - * test computation of (i,j,k) logical coordinates from morton indexes 3D - */ - TEST(sfc, morton3D_getCoordinates){ + * test computation of (i,j,k) logical coordinates from morton indexes 3D + */ + TEST(sfc, morton3D_getCoordinates) + { constexpr int dim = 3; SFC morton; - auto ijk = morton.getCoordinates( static_cast( 1095 ) ); - ASSERT_EQ( ijk(0), 5 ); - ASSERT_EQ( ijk(1), 9 ); - ASSERT_EQ( ijk(2), 1 ); - + auto ijk = morton.getCoordinates(static_cast(1095)); + ASSERT_EQ(ijk(0), 5); + ASSERT_EQ(ijk(1), 9); + ASSERT_EQ(ijk(2), 1); } /* - * test computation of hilbert key for 2D - */ - TEST(sfc, hilbert2D_getKey){ + * test computation of hilbert key for 2D + */ + TEST(sfc, hilbert2D_getKey) + { constexpr int dim = 2; - - SFC hilbert; - Coord_2D_t ij = { 0, 0 }; - ASSERT_EQ( hilbert.getKey( ij ), static_cast( 0 ) ); + SFC hilbert; - ij = { 1, 1 }; - ASSERT_EQ( hilbert.getKey( ij ), static_cast( 2 ) ); + Coord_2D_t ij = {0, 0}; + ASSERT_EQ(hilbert.getKey(ij), static_cast(0)); - ij = { 1, 2 }; - ASSERT_EQ( hilbert.getKey( ij ), static_cast( 7 ) ); + ij = {1, 1}; + ASSERT_EQ(hilbert.getKey(ij), static_cast(2)); - ij = { 3, 1 }; - ASSERT_EQ( hilbert.getKey( ij ), static_cast( 12 ) ); + ij = {1, 2}; + ASSERT_EQ(hilbert.getKey(ij), static_cast(7)); - ij = { 3 , 4 }; - ASSERT_EQ( hilbert.getKey( ij ), static_cast( 31 ) ); + ij = {3, 1}; + ASSERT_EQ(hilbert.getKey(ij), static_cast(12)); - ij = { 3 , 5 }; - ASSERT_EQ( hilbert.getKey( ij ), static_cast( 28 ) ); + ij = {3, 4}; + ASSERT_EQ(hilbert.getKey(ij), static_cast(53)); - ij = { 3, 6 }; - ASSERT_EQ( hilbert.getKey( ij ), static_cast( 27 ) ); + ij = {3, 5}; + ASSERT_EQ(hilbert.getKey(ij), static_cast(52)); + ij = {3, 6}; + ASSERT_EQ(hilbert.getKey(ij), static_cast(51)); } /* - * test computation of hilbert key & back to {i,j,k} - */ + * test computation of hilbert key & back to {i,j,k} + */ // TEST(tests_hilbert, test_consistency_hilbert3D){ - + // { // Logical_Pos_t pt1 {0, 0 ,0}; // auto hk1 = getHilbertKey3D( pt1, 0 ); @@ -173,10 +175,10 @@ namespace samurai // } /* - * test computation of morton indexes 3D - */ + * test computation of morton indexes 3D + */ // TEST(tests_hilbert, test_consistency_hilbert3D_old){ - + // { // auto hk1 = get_hilbert_key_3D( {0, 0 ,0}, 0 ); // auto pt1_r = get_logical_from_hilbert_3D( hk1, 0, 3); From a2ee1a7629cf6924642d2653f919eaeccd1f4bff Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 25 Apr 2024 10:02:27 +0200 Subject: [PATCH 057/170] fix bug template parameter to keep number of ghosts in config --- include/samurai/load_balancing_sfc.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index bc5ffd30d..839108b36 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -34,11 +34,9 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer - Mesh load_balance_impl(Mesh& mesh, Fields&... data) + template + Mesh_t load_balance_impl(Mesh_t& mesh, Fields&... data) { - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; using inter_t = samurai::Interval; using CellList_t = typename Mesh_t::cl_type; using CellArray_t = samurai::CellArray; From ceb78eb597d9ee4d19fddc275a917de2fc35502f Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 25 Apr 2024 15:47:28 +0200 Subject: [PATCH 058/170] update unit test mrmesh --- tests/test_mrmesh.cpp | 78 ++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/tests/test_mrmesh.cpp b/tests/test_mrmesh.cpp index 10be6758c..cc6e46052 100644 --- a/tests/test_mrmesh.cpp +++ b/tests/test_mrmesh.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -11,15 +12,10 @@ namespace samurai { - /* - * test MR Mesh; - */ - TEST(mrmesh, test_nbcells2D) + template + inline samurai::CellList getInitState() { - constexpr int dim = 2; - - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; + assert(dim == 2); samurai::CellList cl; cl[0][{0}].add_interval({0, 4}); @@ -42,21 +38,47 @@ namespace samurai cl[2][{14}].add_interval({14, 16}); cl[2][{15}].add_interval({14, 16}); - Mesh_t mesh(cl, 0, 2); + return cl; + } + + /* + * test MR Mesh; + */ + TEST(mrmesh, test_nbcells2D) + { + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + + Mesh_t mesh(getInitState(), 0, 2); + + samurai::save("./", "test_mrmesh_cells_and_ghosts", mesh); ASSERT_EQ(mesh.min_level(), 0); ASSERT_EQ(mesh.max_level(), 2); - std::vector nCellPerLevel_withGhost = {36, 48, 32}; // including ghost + /** + * Need fix, this does not work. + */ + // std::vector nCellPerLevel_withGhost = {36, 48, 32}; // including ghost + // for (size_t ilvl = mesh.min_level(); ilvl <= mesh.max_level(); ++ilvl) + // { + // std::cerr << "cells_and_ghosts [" << ilvl << "]: " << mesh.nb_cells(ilvl, samurai::MRMeshId::cells_and_ghosts) << std::endl; + // // ASSERT_EQ(mesh.nb_cells(ilvl), nCellPerLevel_withGhost[ilvl]); + // } + + std::vector nCellPerLevel_proj = {5, 2, 0}; // proj cells for (size_t ilvl = mesh.min_level(); ilvl <= mesh.max_level(); ++ilvl) { - ASSERT_EQ(mesh.nb_cells(mesh.min_level()), nCellPerLevel_withGhost[mesh.min_level()]); + // std::cerr << "proj_cells [" << ilvl << "]: " << mesh.nb_cells(ilvl, samurai::MRMeshId::proj_cells) << std::endl; + ASSERT_EQ(mesh.nb_cells(ilvl, samurai::MRMeshId::proj_cells), nCellPerLevel_proj[ilvl]); } std::vector nCellPerLevel_leaves = {11, 18, 8}; // not including ghost for (size_t ilvl = mesh.min_level(); ilvl <= mesh.max_level(); ++ilvl) { - ASSERT_EQ(mesh.nb_cells(mesh.min_level(), samurai::MRMeshId::cells), nCellPerLevel_leaves[mesh.min_level()]); + ASSERT_EQ(mesh.nb_cells(ilvl, samurai::MRMeshId::cells), nCellPerLevel_leaves[ilvl]); } } @@ -64,35 +86,15 @@ namespace samurai { constexpr int dim = 2; - using Config = samurai::MRConfig; - using Mesh_t = samurai::MRMesh; - - samurai::CellList cl; - cl[0][{0}].add_interval({0, 4}); - cl[0][{1}].add_interval({0, 1}); - cl[0][{1}].add_interval({3, 4}); - cl[0][{2}].add_interval({0, 1}); - cl[0][{2}].add_interval({3, 4}); - cl[0][{3}].add_interval({0, 3}); + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using Interval_t = typename Mesh_t::interval_t; - cl[1][{2}].add_interval({2, 6}); - cl[1][{3}].add_interval({2, 6}); - cl[1][{4}].add_interval({2, 4}); - cl[1][{4}].add_interval({5, 6}); - cl[1][{5}].add_interval({2, 6}); - cl[1][{6}].add_interval({6, 8}); - cl[1][{7}].add_interval({6, 7}); - - cl[2][{8}].add_interval({8, 10}); - cl[2][{9}].add_interval({8, 10}); - cl[2][{14}].add_interval({14, 16}); - cl[2][{15}].add_interval({14, 16}); - - Mesh_t mesh(cl, 0, 2); + Mesh_t mesh(getInitState(), 0, 2); - Interval i{0, 3, 0}; + Interval_t i{0, 3, 0}; - // mesh.exists( samurai::MRMeshId::cells, 1, ) + // const auto val = mesh.exists(samurai::MRMeshId::cells, 1, i); } TEST(mrmesh, test_merge) From bc8438b3046c3274c87fc81f1e67bd9a6cfd67b2 Mon Sep 17 00:00:00 2001 From: Loic Gouarin Date: Thu, 2 May 2024 10:39:40 +0200 Subject: [PATCH 059/170] fix: MPI (#199) The multi resolution has been broken with MPI. With this PR, we obtain good results and the same behavior for 1 process and n processes. By submitting this PR, you agree to follow our [Code of Conduct](https://github.com/hpc-maths/samurai/blob/master/docs/CODE_OF_CONDUCT.md) - [x] I agree to follow this project's Code of Conduct --- include/samurai/mesh.hpp | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index ecb2bb587..7b1953a91 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -284,6 +284,8 @@ namespace samurai set_origin_point(origin_point()); set_scaling_factor(scaling_factor()); + update_mesh_neighbour(); + } template @@ -811,21 +813,21 @@ namespace samurai // return neighbour_rank; // }; - static_nested_loop( - [&](auto& shift) - { - if (xt::any(shift)) - { - for (std::size_t d = 0; d < dim; ++d) - { - if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) - { - return; - } - } - m_mpi_neighbourhood.push_back(neighbour(shift)); - } - }); + // static_nested_loop( + // [&](auto& shift) + // { + // if (xt::any(shift)) + // { + // for (std::size_t d = 0; d < dim; ++d) + // { + // if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) + // { + // return; + // } + // } + // m_mpi_neighbourhood.push_back(neighbour(shift)); + // } + // }); #endif } From 6e6579905ed992746d68990248bf93fc7d04c648 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 21 Mar 2024 16:38:52 +0100 Subject: [PATCH 060/170] remove load balancing from mesh class + add merge & remove functions --- include/samurai/mesh.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 7b1953a91..02931c22a 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -365,11 +365,11 @@ namespace samurai m_cells[mesh_id_t::cells] = {cl, false}; - update_mesh_neighbour(); // required to do that here ?? construct_subdomain(); // required ? construct_union(); // required ? update_sub_mesh(); // perform MPI allReduce calls renumbering(); // required ? + update_mesh_neighbour(); // required to do that here ?? } template @@ -828,7 +828,6 @@ namespace samurai // m_mpi_neighbourhood.push_back(neighbour(shift)); // } // }); - #endif } @@ -865,7 +864,11 @@ namespace samurai // remove cells cl_type cl; size_t diff_ncells = 0; +<<<<<<< HEAD for (size_t ilvl = refmesh.min_level(); ilvl <= refmesh.max_level(); ++ilvl) +======= + for (int ilvl = refmesh.min_level(); ilvl <= refmesh.max_level(); ++ilvl) +>>>>>>> fa278dd (remove load balancing from mesh class + add merge & remove functions) { auto diff = samurai::difference(refmesh[ilvl], lca[ilvl]); From 317c21224640d22062dfee2fca894c34d5671934 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Apr 2024 15:59:58 +0200 Subject: [PATCH 061/170] fix warnings --- include/samurai/mesh.hpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 02931c22a..d6746a41c 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -864,11 +864,7 @@ namespace samurai // remove cells cl_type cl; size_t diff_ncells = 0; -<<<<<<< HEAD for (size_t ilvl = refmesh.min_level(); ilvl <= refmesh.max_level(); ++ilvl) -======= - for (int ilvl = refmesh.min_level(); ilvl <= refmesh.max_level(); ++ilvl) ->>>>>>> fa278dd (remove load balancing from mesh class + add merge & remove functions) { auto diff = samurai::difference(refmesh[ilvl], lca[ilvl]); @@ -878,6 +874,13 @@ namespace samurai cl[ilvl][index].add_interval(interval); diff_ncells += interval.size(); }); + + diff( + [&](auto& interval, auto& index) + { + cl[ilvl][index].add_interval(interval); + diff_ncells += interval.size(); + }); } // new mesh for current process From b791f24fa4ef80afed5130ad4e0f17112f61676e Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 18 Apr 2024 17:05:07 +0200 Subject: [PATCH 062/170] fix update neighbour in constructor --- include/samurai/mesh.hpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index d6746a41c..270292ab8 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -874,13 +874,6 @@ namespace samurai cl[ilvl][index].add_interval(interval); diff_ncells += interval.size(); }); - - diff( - [&](auto& interval, auto& index) - { - cl[ilvl][index].add_interval(interval); - diff_ncells += interval.size(); - }); } // new mesh for current process From 30eac01301647f967afb15b6e8c0a9f402420041 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 2 May 2024 14:10:46 +0200 Subject: [PATCH 063/170] remove check nan --- include/samurai/load_balancing.hpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 3d3193f0d..7c6defb12 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -300,12 +300,7 @@ namespace samurai boost::mpi::communicator world; Field_t new_field("new_f", new_mesh); - -#ifdef SAMURAI_CHECK_NAN - new_field.fill(std::nan("")); -#else new_field.fill(0); -#endif auto& mesh = field.mesh(); @@ -361,7 +356,7 @@ namespace samurai auto in_interface = intersection(neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); in_interface( - [&](const auto& i, const auto& index) + [&]( [[maybe_unused]]const auto& i, [[maybe_unused]]const auto& index) { isintersect = true; }); From b2da43ae0d5f6bd0f83ff27eff73fd4ace7f281f Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 2 May 2024 14:11:28 +0200 Subject: [PATCH 064/170] remove useless argument --- include/samurai/load_balancing_sfc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 839108b36..b4f57c3a9 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -35,7 +35,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer - Mesh_t load_balance_impl(Mesh_t& mesh, Fields&... data) + Mesh_t load_balance_impl( Mesh_t& mesh ) { using inter_t = samurai::Interval; using CellList_t = typename Mesh_t::cl_type; From c676532fe3fc6b3779c1a3a5467b3cebce44f88c Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 2 May 2024 14:16:34 +0200 Subject: [PATCH 065/170] add mrmesh tests --- tests/test_mrmesh.cpp | 93 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/tests/test_mrmesh.cpp b/tests/test_mrmesh.cpp index cc6e46052..62f0f427f 100644 --- a/tests/test_mrmesh.cpp +++ b/tests/test_mrmesh.cpp @@ -1,11 +1,9 @@ #include -#include -#include #include -#include #include #include +#include #include @@ -155,4 +153,93 @@ namespace samurai ASSERT_EQ(mesh.nb_cells(Mesh_t::mesh_id_t::cells), 9); } + TEST(mrmesh, test_construct_subdomain) + { + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + + samurai::CellList cl; + cl[0][{0}].add_interval({0, 4}); + cl[0][{1}].add_interval({0, 1}); + cl[0][{1}].add_interval({3, 4}); + cl[0][{2}].add_interval({0, 1}); + cl[0][{2}].add_interval({3, 4}); + + cl[1][{2}].add_interval({2, 6}); + cl[1][{3}].add_interval({2, 6}); + cl[1][{4}].add_interval({2, 4}); + cl[1][{4}].add_interval({5, 6}); + cl[1][{5}].add_interval({2, 6}); + cl[1][{6}].add_interval({6, 8}); + cl[1][{7}].add_interval({6, 7}); + + Mesh_t mesh( cl, 0, 1); + + bool intersect_found = false; + const auto & subdom = mesh.subdomain(); + for(std::size_t level=mesh.min_level(); level 0) { intersect_found = true; } + } + + ASSERT_FALSE( intersect_found ); + + } + + TEST(mrmesh, test_construct_union) + { + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + using cl_t = Mesh_t::cl_type; + using ca_t = Mesh_t::ca_type; + + Mesh_t mesh( getInitState(), 0, 2); + + auto un = mesh.get_union(); + + ASSERT_EQ( un[ 0 ].nb_cells(), 5); + { + cl_t cl; + cl[0][{1}].add_interval({1,3}); + cl[0][{2}].add_interval({1,3}); + cl[0][{3}].add_interval({3,4}); + ca_t ca = { cl, false }; + auto tmp_ = samurai::difference( ca[0], un[0] ); + + size_t ni = 0; + tmp_( [&]([[maybe_unused]]const auto & interval, [[maybe_unused]]const auto & index){ + ni +=1; + }); + ASSERT_EQ( ni, 0); + } + + ASSERT_EQ( un[ 1 ].nb_cells(), 2); + { + cl_t cl; + cl[1][{4}].add_interval({4,5}); + cl[1][{7}].add_interval({7,8}); + ca_t ca = { cl, false }; + auto tmp_ = samurai::difference( ca[1], un[1] ); + + size_t ni = 0; + tmp_( [&]([[maybe_unused]]const auto & interval, [[maybe_unused]]const auto & index){ + ni +=1; + }); + ASSERT_EQ( ni, 0); + } + + ASSERT_EQ( un[ 2 ].nb_cells(), 0); + + + } + } \ No newline at end of file From b1d580552d407544f3d7fb4325665ba26465483b Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 2 May 2024 14:29:56 +0200 Subject: [PATCH 066/170] fix example with load balancing --- demos/FiniteVolume/advection_2d.cpp | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 9e2dd3384..0be8cf0da 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -207,6 +207,7 @@ int main(int argc, char* argv[]) fs::path path = fs::current_path(); std::string filename = "FV_advection_2d"; std::size_t nfiles = 1; + int nt_loadbalance = 10; // nombre d'iteratio entre les equilibrages app.add_option("--min-corner", min_corner, "The min corner of the box")->capture_default_str()->group("Simulation parameters"); app.add_option("--max-corner", max_corner, "The max corner of the box")->capture_default_str()->group("Simulation parameters"); @@ -215,6 +216,7 @@ int main(int argc, char* argv[]) app.add_option("--Tf", Tf, "Final time")->capture_default_str()->group("Simulation parameters"); app.add_option("--min-level", min_level, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); app.add_option("--max-level", max_level, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--nt-loadbalance", nt_loadbalance, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); app.add_option("--mr-eps", mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") ->capture_default_str() ->group("Multiresolution"); @@ -257,18 +259,16 @@ int main(int argc, char* argv[]) while (t != Tf) { - if (nt % 20 == 0 && nt > 1) + if (nt % nt_loadbalance == 0 && nt > 1 ) { std::cout << "\t> Load balancing mesh ... " << std::endl; + myTimers.start("load-balancing"); balancer.load_balance(mesh, u); myTimers.stop("load-balancing"); - // const std::string suffix = fmt::format("_loadbalanced_{}", nt); - // save(path, filename, u, suffix); - - // samurai::finalize(); - // return 0; + std::string suffix = fmt::format("_loadbalanced_bf_{}", nt); + save( path, filename, u, suffix); } myTimers.start("MRadaptation"); @@ -302,11 +302,9 @@ int main(int argc, char* argv[]) std::swap(u.array(), unp1.array()); myTimers.start("I/O"); - // if (t >= static_cast(nsave + 1) * dt_save || t == Tf || true ) - if (nt % 20 == 0) + if (t >= static_cast(nsave + 1) * dt_save || t == Tf || true ) { - // const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; - const std::string suffix = fmt::format("_ite_{}", nt); + const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; save(path, filename, u, suffix); } myTimers.stop("I/O"); From 6170114b0a452527d953e729d50c98c62775eea4 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 3 May 2024 10:47:17 +0200 Subject: [PATCH 067/170] fix load balance diffusion gravity --- .../samurai/load_balancing_diffusion_cell.hpp | 55 ++++++++++++++----- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 63073281b..c75b2edb9 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -46,7 +46,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer - void load_balance_impl( Mesh_t & mesh ){ + Mesh_t load_balance_impl( Mesh_t & mesh ){ using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; using CellList_t = typename Mesh_t::cl_type; @@ -214,7 +214,11 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( new_mesh ); + // samurai::discover_neighbour( new_mesh ); - samurai::discover_neighbour( mesh ); - samurai::discover_neighbour( mesh ); - + return new_mesh; } }; \ No newline at end of file From acf313e550cb55e58659057ed7f7df27472b5017 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 3 May 2024 10:47:53 +0200 Subject: [PATCH 068/170] update load balance sfc --- include/samurai/load_balancing_sfc.hpp | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index b4f57c3a9..c83e9d48c 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -34,7 +34,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer + template Mesh_t load_balance_impl( Mesh_t& mesh ) { using inter_t = samurai::Interval; @@ -79,7 +79,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer> ijk; for (size_t idim = 0; idim < dim; ++idim) { @@ -125,10 +125,9 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancersecond.given) @@ -248,13 +247,11 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Number of cells in OLD mesh(mesh_id_t::cells) : " << mesh.nb_cells(Mesh_t::mesh_id_t::cells) << std::endl; - logs << "\t> Number of cells in NEW mesh(mesh_id_t::cells) : " << new_mesh.nb_cells(Mesh_t::mesh_id_t::cells) << std::endl; - - logs << "\t> Number of cells in OLD mesh(mesh_id_t::all) : " << mesh.nb_cells() << std::endl; - logs << "\t> Number of cells in NEW mesh(mesh_id_t::all) : " << new_mesh.nb_cells() << std::endl; + Mesh_t new_mesh( new_cl, mesh ); // update neighbour connectivity // samurai::discover_neighbour(new_mesh); From 0816dffb52101dd72b3fb1fa9e8a059d316ac234 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 6 May 2024 12:35:10 +0200 Subject: [PATCH 069/170] fix return value for interface propagation --- .../load_balancing_diffusion_interval.hpp | 71 ++++++++++++++++--- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index b4fc041f5..74c305eb0 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -82,11 +82,12 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer - void load_balance_impl( Mesh_t & mesh ){ + Mesh_t load_balance_impl( Mesh_t & mesh ){ using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; using CellList_t = typename Mesh_t::cl_type; using CellArray_t = samurai::CellArray; + using mesh_id_t = typename Mesh_t::mesh_id_t; boost::mpi::communicator world; @@ -124,12 +125,17 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer= 0 ) continue; + + // FIX: Add condition (&& interface not empty ) ? + // Neighbourhood should have been updated in a way that an interface exists ! if( - new_fluxes[ nbi ] > requested_load ){ requested_load = - new_fluxes[ nbi ]; requester = nbi; @@ -163,7 +169,9 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer Process {}, Warning no interface with a requester neighbour # {}", world.rank(), + neighbourhood[ requester ].rank ) << std::endl; logs << "Requester neighbour, no interface found, set fluxes to " << new_fluxes[ requester ] << std::endl; } }else{ @@ -207,6 +215,12 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer 0 ) { // receive data - samurai::CellArray to_rcv; + CellArray_t to_rcv; world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); - mesh.merge( to_rcv ); + + logs << fmt::format("\t> Rank # {} receiving {} cells from rank # {}", world.rank(), to_rcv.nb_cells(), neighbourhood[ ni ].rank) << std::endl; + + // old strategy + // mesh.merge( to_rcv ); + + // add to future mesh of current process + samurai::for_each_interval(to_rcv, + [&](std::size_t level, const auto& interval, const auto& index) + { + new_cl[ level ][ index ].add_interval( interval ); + }); + }else{ // send data to CellArray_t to_send = { cl_to_send[ ni ], false }; world.send( neighbourhood[ ni ].rank, 42, to_send ); + + logs << fmt::format("\t> Rank # {} sending {} cells to rank # {}", world.rank(), to_send.nb_cells(), neighbourhood[ ni ].rank) << std::endl; + + samurai::for_each_interval( to_send, + [&](std::size_t level, const auto& interval, const auto& index) + { + need_remove[ level ][ index ].add_interval( interval ); + }); } } - // update neighbour mesh - this should end up with the same result but .. - mesh.update_mesh_neighbour(); + /* ---------------------------------------------------------------------------------------------------------- */ + /* ------- Construct new mesh for current process ----------------------------------------------------------- */ + /* ---------------------------------------------------------------------------------------------------------- */ + + // add to new_cl interval that were not sent + CellArray_t need_remove_ca = { need_remove }; // to optimize + for( size_t level=mesh.min_level(); level<=mesh.max_level(); ++level ){ + auto diff = samurai::difference( mesh[ mesh_id_t::cells ][ level ], need_remove_ca[ level ] ); + + diff([&]( auto & interval, auto & index ){ + new_cl[ level ][ index ].add_interval( interval ); + }); + } + + Mesh_t new_mesh( new_cl, mesh ); + + logs << fmt::format("Rank # {} old mesh {} nb cells", world.rank(), mesh.nb_cells() ) << std::endl; + logs << fmt::format("Rank # {} new mesh {} nb cells", world.rank(), new_mesh.nb_cells() ) << std::endl; // update neighbour connectivity - samurai::discover_neighbour( mesh ); - samurai::discover_neighbour( mesh ); + // samurai::discover_neighbour( mesh ); + // samurai::discover_neighbour( mesh ); + + samurai::save("./", "new-mesh-LB", new_mesh); + return new_mesh; } template From bcf319415c514debc34937f61539a26abb50ab18 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 6 May 2024 15:42:04 +0200 Subject: [PATCH 070/170] fix interval propagation --- .../load_balancing_diffusion_interval.hpp | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index 74c305eb0..aeaf058aa 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -200,8 +200,9 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer Rank # {} receiving {} cells from rank # {}", world.rank(), to_rcv.nb_cells(), neighbourhood[ ni ].rank) << std::endl; + logs << fmt::format("\t>[load_balance_impl]::interval Rank # {} receiving {} cells from rank # {}", world.rank(), to_rcv.nb_cells(), neighbourhood[ ni ].rank) << std::endl; // old strategy // mesh.merge( to_rcv ); @@ -247,7 +248,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer Rank # {} sending {} cells to rank # {}", world.rank(), to_send.nb_cells(), neighbourhood[ ni ].rank) << std::endl; + logs << fmt::format("\t>[load_balance_impl]::interval Rank # {} sending {} cells to rank # {}", world.rank(), to_send.nb_cells(), neighbourhood[ ni ].rank) << std::endl; samurai::for_each_interval( to_send, [&](std::size_t level, const auto& interval, const auto& index) @@ -255,6 +256,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer( mesh ); - // samurai::discover_neighbour( mesh ); - - samurai::save("./", "new-mesh-LB", new_mesh); - return new_mesh; } template + [[deprecated]] void load_balance_impl2( Mesh_t & mesh ){ using interval_t = typename Mesh_t::interval_t; From df233043c08dd27f8da93d5133b4ba28d7b68020 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 6 May 2024 15:43:33 +0200 Subject: [PATCH 071/170] fix: discover neighbour after field update --- include/samurai/load_balancing.hpp | 77 ++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 7c6defb12..c0ae8b014 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -299,61 +299,75 @@ namespace samurai using value_t = typename Field_t::value_type; boost::mpi::communicator world; + std::ofstream logs; + logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs << fmt::format("> [LoadBalancer]::update_field rank # {} -> '{}' ", world.rank(), field.name() ) << std::endl; + Field_t new_field("new_f", new_mesh); new_field.fill(0); - auto& mesh = field.mesh(); + auto & old_mesh = field.mesh(); // auto min_level = boost::mpi::all_reduce(world, mesh[mesh_id_t::cells].min_level(), boost::mpi::minimum()); // auto max_level = boost::mpi::all_reduce(world, mesh[mesh_id_t::cells].max_level(), boost::mpi::maximum()); - auto min_level = mesh.min_level(); - auto max_level = mesh.max_level(); + auto min_level = old_mesh.min_level(); + auto max_level = old_mesh.max_level(); + // copy data of intervals that are didn't move for (std::size_t level = min_level; level <= max_level; ++level) { - auto set = intersection(mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); - set.apply_op(copy(new_field, field)); + auto intersect_old_new = intersection(old_mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + intersect_old_new.apply_op( samurai::copy( new_field, field ) ); } - std::vector req; + logs << fmt::format("> [LoadBalancer]::update_field rank # {}: data copied for intersection old/new ", world.rank() ) << std::endl; + + std::vector req; std::vector> to_send(new_mesh.mpi_neighbourhood().size()); std::size_t i_neigh = 0; + // build payload of field that has been sent to neighbour, so compare old mesh with new neighbour mesh for (auto& neighbour : new_mesh.mpi_neighbourhood()) { + auto & neighbour_new_mesh = neighbour.mesh; for (std::size_t level = min_level; level <= max_level; ++level) { - if (!mesh[mesh_id_t::cells][level].empty() && !neighbour.mesh[mesh_id_t::cells][level].empty()) + if (!old_mesh[mesh_id_t::cells][level].empty() && !neighbour_new_mesh[mesh_id_t::cells][level].empty()) { - auto out_interface = intersection(mesh[mesh_id_t::cells][level], neighbour.mesh[mesh_id_t::cells][level]); - out_interface( - [&](const auto& i, const auto& index) - { - std::copy(field(level, i, index).begin(), field(level, i, index).end(), std::back_inserter(to_send[i_neigh])); - // std::cerr << fmt::format("Process {}, send interval {}", world.rank(), i) << std::endl; + auto intersect_old_mesh_new_neigh = intersection( old_mesh[mesh_id_t::cells][level], neighbour_new_mesh[mesh_id_t::cells][level] ); + intersect_old_mesh_new_neigh( + [&](const auto & interval, const auto & index) + { + std::copy(field(level, interval, index).begin(), field(level, interval, index).end(), std::back_inserter(to_send[i_neigh])); }); } } if (to_send[i_neigh].size() != 0) { - req.push_back(world.isend(neighbour.rank, neighbour.rank, to_send[i_neigh++])); + req.push_back( world.isend( neighbour.rank, neighbour.rank, to_send[ i_neigh ] ) ); + i_neigh ++; + + logs << fmt::format("> [LoadBalancer]::update_field rank # {}: data to send to {}", world.rank(), neighbour.rank ) << std::endl; } } - for (auto& neighbour : mesh.mpi_neighbourhood()) + logs << fmt::format("> [LoadBalancer]::update_field rank # {}, nb req isend {}", world.rank(), req.size() ) << std::endl; + + // build payload of field that I need to receive from neighbour, so compare NEW mesh with OLD neighbour mesh + for (auto& old_neighbour : old_mesh.mpi_neighbourhood()) { bool isintersect = false; for (std::size_t level = min_level; level <= max_level; ++level) { - if (!new_mesh[mesh_id_t::cells][level].empty() && !neighbour.mesh[mesh_id_t::cells][level].empty()) + if (!new_mesh[mesh_id_t::cells][level].empty() && !old_neighbour.mesh[mesh_id_t::cells][level].empty()) { std::vector to_recv; std::ptrdiff_t count = 0; - auto in_interface = intersection(neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + auto in_interface = intersection(old_neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); in_interface( [&]( [[maybe_unused]]const auto& i, [[maybe_unused]]const auto& index) @@ -372,13 +386,13 @@ namespace samurai { std::ptrdiff_t count = 0; std::vector to_recv; - world.recv(neighbour.rank, world.rank(), to_recv); + world.recv(old_neighbour.rank, world.rank(), to_recv); for (std::size_t level = min_level; level <= max_level; ++level) { - if (!new_mesh[mesh_id_t::cells][level].empty() && !neighbour.mesh[mesh_id_t::cells][level].empty()) + if (!new_mesh[mesh_id_t::cells][level].empty() && !old_neighbour.mesh[mesh_id_t::cells][level].empty()) { - auto in_interface = intersection(neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + auto in_interface = intersection(old_neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); in_interface( [&](const auto& i, const auto& index) @@ -421,19 +435,30 @@ namespace samurai template void load_balance(Mesh& mesh, Field_t& field, Fields&... kw) { - // build new mesh using "Flavor" load balancing strategy + // specific load balancing strategy auto new_mesh = static_cast(this)->load_balance_impl(mesh); - // Manage physical fields on new load balanced mesh + // update each physical field on the new load balanced mesh + SAMURAI_TRACE("[LoadBalancer::load_balance]::Updating fields ... "); update_fields(new_mesh, field, kw...); - // std::swap(mesh, new_mesh); - - // swapping meshes + // swap mesh reference to new load balanced mesh. FIX: this is not clean + SAMURAI_TRACE("[LoadBalancer::load_balance]::Swapping meshes ... "); field.mesh().swap(new_mesh); - // save("load_balance", mesh, field); + + // discover neighbours: add new neighbours if a new interface appears or remove old neighbours + discover_neighbour< static_cast(Mesh::dim) >( new_mesh ); + discover_neighbour< static_cast(Mesh::dim) >( new_mesh ); } + /** + * Try to evaluate / compute a load balancing score. We expect from a good load + * balancing strategy to: + * - optimize the number of neighbours (reduce comm.) + * - optimize the number of ghosts cells (reduce comm.) + * - load balance charge between processes + * - optimize the size of interval (samurai specific, expect better perf, better simd) + */ template void evaluate_balancing(Mesh& mesh) const { From e8e2164dce7a813b3f19e2ddc698f83b79eeca5f Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 6 May 2024 15:43:56 +0200 Subject: [PATCH 072/170] remove discover for SFC --- include/samurai/load_balancing_sfc.hpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index c83e9d48c..1ec0d2005 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -253,10 +253,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer(new_mesh); - // samurai::discover_neighbour(new_mesh); - return new_mesh; } }; \ No newline at end of file From 54dad07a7c72ebb2e177729d2e00b26740802d45 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 7 May 2024 09:08:17 +0200 Subject: [PATCH 073/170] undo partition_mesh all processes as neighbours --- include/samurai/mesh.hpp | 68 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 270292ab8..4426b19f9 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -790,44 +790,44 @@ namespace samurai this->m_cells[mesh_id_t::cells][start_level] = {start_level, subdomain_box}; **/ - m_mpi_neighbourhood.reserve(static_cast(size) - 1); - for (int ir = 0; ir < size; ++ir) - { - if (ir != rank) - { - m_mpi_neighbourhood.push_back(ir); - } - } - - // // Neighbours - // m_mpi_neighbourhood.reserve(static_cast(pow(3, dim) - 1)); - // auto neighbour = [&](xt::xtensor_fixed> shift) + // m_mpi_neighbourhood.reserve(static_cast(size) - 1); + // for (int ir = 0; ir < size; ++ir) // { - // auto neighbour_rank = rank; - // int product_of_preceding_sizes = 1; - // for (std::size_t d = 0; d < dim; ++d) + // if (ir != rank) // { - // neighbour_rank += product_of_preceding_sizes * shift[d]; - // product_of_preceding_sizes *= sizes[d]; + // m_mpi_neighbourhood.push_back(ir); // } - // return neighbour_rank; - // }; + // } - // static_nested_loop( - // [&](auto& shift) - // { - // if (xt::any(shift)) - // { - // for (std::size_t d = 0; d < dim; ++d) - // { - // if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) - // { - // return; - // } - // } - // m_mpi_neighbourhood.push_back(neighbour(shift)); - // } - // }); + // // Neighbours + m_mpi_neighbourhood.reserve(static_cast(pow(3, dim) - 1)); + auto neighbour = [&](xt::xtensor_fixed> shift) + { + auto neighbour_rank = rank; + int product_of_preceding_sizes = 1; + for (std::size_t d = 0; d < dim; ++d) + { + neighbour_rank += product_of_preceding_sizes * shift[d]; + product_of_preceding_sizes *= sizes[d]; + } + return neighbour_rank; + }; + + static_nested_loop( + [&](auto& shift) + { + if (xt::any(shift)) + { + for (std::size_t d = 0; d < dim; ++d) + { + if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) + { + return; + } + } + m_mpi_neighbourhood.push_back(neighbour(shift)); + } + }); #endif } From 13894db2e047deb83d0c14c24869c19666e15594 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 7 May 2024 09:08:56 +0200 Subject: [PATCH 074/170] fix gravity based to take load evolution into account --- .../samurai/load_balancing_diffusion_cell.hpp | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index c75b2edb9..f8fe88119 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -91,6 +91,14 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( mesh ); + std::vector ncells_interface ( interface.size(), 0 ); + for(int ni=0; ni( mesh[ mesh_id_t::cells ] ); @@ -140,6 +148,8 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( world.rank() ) ]; double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; + std::vector mload( world.size(), 0 ); + // select the neighbour for( std::size_t ni=0; ni( mid_point, barycenter_neighbours[ ni ] ) ); // double dist = samurai::getDistance( cell, barycenter_interface_neighbours[ ni ] ) / loads[ neighbour_rank ]; - double coeff = sv[ ni ] / loads[ neighbour_rank ]; // / sv[ ni ]; + double coeff = sv[ ni ] / ( loads[ neighbour_rank ] + mload[ neighbour_rank ] ) ; // / sv[ ni ]; double dist = samurai::getDistance( cell.center(), barycenter_neighbours[ ni ] ) * coeff; - // double dist = std::max( d1, d2 ); - - if( dist < winner_dist ){ + if( dist < winner_dist && ncells_interface[ ni ] > 0 ){ winner_id = static_cast( ni ); winner_dist = dist; + + mload[ neighbour_rank ] += 1; } } @@ -273,10 +283,6 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( new_mesh ); - // samurai::discover_neighbour( new_mesh ); return new_mesh; } From 7e712c8dd59c60c8079e936ece77dadfefeefcc4 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 7 May 2024 09:09:20 +0200 Subject: [PATCH 075/170] add new LB algorithm based on field --- include/samurai/load_balancing_diffusion.hpp | 240 +++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 include/samurai/load_balancing_diffusion.hpp diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp new file mode 100644 index 000000000..158c93a52 --- /dev/null +++ b/include/samurai/load_balancing_diffusion.hpp @@ -0,0 +1,240 @@ +#pragma once + +#include +#include "load_balancing.hpp" +#include + +namespace load_balancing{ + + class Diffusion : public samurai::LoadBalancer { + + private: + int _ndomains; + int _rank; + + public: + + Diffusion() { + + #ifdef SAMURAI_WITH_MPI + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); + #else + _ndomains = 1; + _rank = 0; + #endif + } + + inline std::string getName() const { return "diffusion"; } + + template + Mesh_t load_balance_impl( Mesh_t & mesh ){ + + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; + using mesh_id_t = typename Mesh_t::mesh_id_t; + + using Coord_t = xt::xtensor_fixed>; + + constexpr int dim = static_cast( Mesh_t::dim ); + + boost::mpi::communicator world; + + // For debug + std::ofstream logs; + logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + logs << fmt::format("> New load-balancing using {} ", getName() ) << std::endl; + + // compute fluxes in terms of number of intervals to transfer/receive + std::vector fluxes = samurai::cmptFluxes( mesh ); + std::vector new_fluxes( fluxes ); + + // get loads from everyone + std::vector loads; + int my_load = static_cast( samurai::cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load, loads ); + + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + size_t n_neighbours = neighbourhood.size(); + + { // some debug info + logs << "load : " << my_load << std::endl; + logs << "nneighbours : " << n_neighbours << std::endl; + logs << "neighbours : "; + for( size_t in=0; in cl_to_send( n_neighbours ); + + auto flags = samurai::make_field("diffusion_flag", mesh); + for_each_interval( mesh[ mesh_id_t::cells ], [&]( std::size_t level, const auto & interval, const auto & index ){ + flags( level, interval, index ) = _rank; + }); + + + + bool balancing_done = false; + while( ! balancing_done ){ + + // select neighbour with the highest needs of load + bool neighbour_found = false; + std::size_t requester = 0; + int requested_load = 0; + for(std::size_t nbi=0; nbi= 0 ) continue; + + // FIX: Add condition (&& interface not empty ) ? + // Neighbourhood should have been updated in a way that an interface exists ! + if( - new_fluxes[ nbi ] > requested_load ){ + requested_load = - new_fluxes[ nbi ]; + requester = nbi; + neighbour_found = true; + } + } + + if( ! neighbour_found ){ + { // debug + logs << "No more neighbour found " << std::endl; + } + + balancing_done = true; + break; + } + + { // debug + logs << "Requester neighbour : " << neighbourhood[ requester ].rank << ", fluxes : " << requested_load << std::endl; + } + + // select interval for this neighbour by moving in cartesian direction by one + auto interface = samurai::cmptInterface( mesh, neighbourhood[ requester ].mesh ); + + { // check emptyness of interface, if it is empty, then set fluxes for this neighbour to 0 + size_t nelement = 0; + samurai::for_each_interval( interface, [&]( [[maybe_unused]] std::size_t level, [[maybe_unused]] const auto & interval, + [[maybe_unused]] const auto & index ){ + nelement += 1; + }); + + if( nelement == 0 ){ + new_fluxes[ requester ] = 0; + + { // debug + std::cerr << fmt::format("\t> Process {}, Warning no interface with a requester neighbour # {}", world.rank(), + neighbourhood[ requester ].rank ) << std::endl; + logs << "Requester neighbour, no interface found, set fluxes to " << new_fluxes[ requester ] << std::endl; + } + }else{ + { // debug + logs << "Requester neighbour, interface " << nelement << " intervals " << std::endl; + } + } + } + + // go through interval on the interface and add as much as possible + // skip this to add the whole interface + CellList_t cl_for_neighbour; + + size_t nbIntervalAdded = 0; + samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ + + if( new_fluxes[ requester ] < 0 ){ + cl_for_neighbour[ level ][ index ].add_interval( interval ); + // new_fluxes[ requester ] += 1; // here interval load == 1, no weight; + nbIntervalAdded += 1; + } + + }); + + new_fluxes[ requester ] += nbIntervalAdded; + + { // remove interval from current process mesh and add it to neighbour mesh local copy ! + CellArray_t ca_for_neighbour = { cl_for_neighbour, false }; + + // mesh.remove( ca_for_neighbour ); + // neighbourhood[ requester ].mesh.merge( ca_for_neighbour ); + + // update gobal list that will be sent to neighbour process + samurai::for_each_interval( ca_for_neighbour, [&]( std::size_t level, const auto & interval, const auto & index ){ + cl_to_send[ requester ][ level ][ index ].add_interval( interval ); + }); + } + + { // debug + logs << "New flux for this neighbour : " << new_fluxes[ requester ] << std::endl; + } + + } + + /* ---------------------------------------------------------------------------------------------------------- */ + /* ------- Data transfer between processes ------------------------------------------------------------------ */ + /* ---------------------------------------------------------------------------------------------------------- */ + + CellList_t new_cl, need_remove; + + // at this point local mesh of neighbour are modified ( technically it should match what would results from send) + // and local mesh has been modified + for(std::size_t ni=0; ni 0 ) { // receive data + CellArray_t to_rcv; + world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); + + logs << fmt::format("\t>[load_balance_impl]::interval Rank # {} receiving {} cells from rank # {}", world.rank(), to_rcv.nb_cells(), neighbourhood[ ni ].rank) << std::endl; + + // old strategy + // mesh.merge( to_rcv ); + + // add to future mesh of current process + samurai::for_each_interval(to_rcv, + [&](std::size_t level, const auto& interval, const auto& index) + { + new_cl[ level ][ index ].add_interval( interval ); + }); + + }else{ // send data to + CellArray_t to_send = { cl_to_send[ ni ], false }; + world.send( neighbourhood[ ni ].rank, 42, to_send ); + + logs << fmt::format("\t>[load_balance_impl]::interval Rank # {} sending {} cells to rank # {}", world.rank(), to_send.nb_cells(), neighbourhood[ ni ].rank) << std::endl; + + samurai::for_each_interval( to_send, + [&](std::size_t level, const auto& interval, const auto& index) + { + need_remove[ level ][ index ].add_interval( interval ); + }); + } + + } + + /* ---------------------------------------------------------------------------------------------------------- */ + /* ------- Construct new mesh for current process ----------------------------------------------------------- */ + /* ---------------------------------------------------------------------------------------------------------- */ + + // add to new_cl interval that were not sent + CellArray_t need_remove_ca = { need_remove }; // to optimize + for( size_t level=mesh.min_level(); level<=mesh.max_level(); ++level ){ + auto diff = samurai::difference( mesh[ mesh_id_t::cells ][ level ], need_remove_ca[ level ] ); + + diff([&]( auto & interval, auto & index ){ + new_cl[ level ][ index ].add_interval( interval ); + }); + } + + Mesh_t new_mesh( new_cl, mesh ); + + return new_mesh; + } + + }; +} \ No newline at end of file From 7a3faf47d1b035f3f14fd8765ab44e2fb6bd7097 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 7 May 2024 09:09:56 +0200 Subject: [PATCH 076/170] fix +1 in logical coord --- include/samurai/load_balancing_sfc.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 1ec0d2005..995919f05 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -76,7 +76,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Date: Tue, 7 May 2024 09:10:26 +0200 Subject: [PATCH 077/170] test new version LB --- demos/FiniteVolume/advection_2d.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 0be8cf0da..57d68c970 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -17,6 +17,9 @@ #include #include +#include +#include +#include #include @@ -255,7 +258,10 @@ int main(int argc, char* argv[]) std::size_t nsave = 1; std::size_t nt = 0; - SFC_LoadBalancer_interval balancer; + // SFC_LoadBalancer_interval balancer; + // Diffusion_LoadBalancer_cell balancer; + // Diffusion_LoadBalancer_interval balancer; + load_balancing::Diffusion balancer; while (t != Tf) { @@ -263,11 +269,14 @@ int main(int argc, char* argv[]) { std::cout << "\t> Load balancing mesh ... " << std::endl; + std::string suffix = fmt::format("_loadbalanced_bf_{}", nt); + save( path, filename, u, suffix); + myTimers.start("load-balancing"); balancer.load_balance(mesh, u); myTimers.stop("load-balancing"); - std::string suffix = fmt::format("_loadbalanced_bf_{}", nt); + suffix = fmt::format("_loadbalanced_af_{}", nt); save( path, filename, u, suffix); } From 23f8cd88b80b782ceb258195fdd7626ddb64f805 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 7 May 2024 22:22:42 +0200 Subject: [PATCH 078/170] fix unit test load balancing --- tests/test_loadbalancing.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_loadbalancing.cpp b/tests/test_loadbalancing.cpp index 0ead91a76..5d2a52bcc 100644 --- a/tests/test_loadbalancing.cpp +++ b/tests/test_loadbalancing.cpp @@ -6,12 +6,15 @@ #include #include +#ifdef SAMURAI_WITH_MPI #include +#endif #include namespace samurai { +#ifdef SAMURAI_WITH_MPI /* * test cmptLoad; */ @@ -85,5 +88,6 @@ namespace samurai { ASSERT_EQ( interface.nb_cells(), 10 ); } - + +#endif } From 9d03b68f00d1138b5deb02c2e7d7c0b13057b005 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 7 May 2024 22:23:43 +0200 Subject: [PATCH 079/170] fix load balancing dim parameter + weight for bayrcenter --- include/samurai/load_balancing.hpp | 330 +++++++++-------------------- 1 file changed, 100 insertions(+), 230 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index c0ae8b014..be9406e24 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -12,69 +12,71 @@ #include // statistics -#include +#ifdef WITH_STATS +#include +#endif namespace samurai { - namespace load_balance - { + // namespace load_balance + // { - template - Mesh_t merge(Mesh_t& mesh, const CellArray_t& lca) - { - using cl_type = typename Mesh_t::cl_type; + // template + // Mesh_t merge(Mesh_t& mesh, const CellArray_t& lca) + // { + // using cl_type = typename Mesh_t::cl_type; - auto& refmesh = mesh[Mesh_t::mesh_id_t::cells]; + // auto& refmesh = mesh[Mesh_t::mesh_id_t::cells]; - auto minlevel = std::min(refmesh.min_level(), lca.min_level()); - auto maxlevel = std::max(refmesh.max_level(), lca.max_level()); + // auto minlevel = std::min(refmesh.min_level(), lca.min_level()); + // auto maxlevel = std::max(refmesh.max_level(), lca.max_level()); - cl_type cl; - for (size_t ilvl = minlevel; ilvl <= maxlevel; ++ilvl) - { - auto un = samurai::union_(refmesh[ilvl], lca[ilvl]); + // cl_type cl; + // for (size_t ilvl = minlevel; ilvl <= maxlevel; ++ilvl) + // { + // auto un = samurai::union_(refmesh[ilvl], lca[ilvl]); - un( - [&](auto& interval, auto& indices) - { - cl[ilvl][indices].add_interval(interval); - }); - } + // un( + // [&](auto& interval, auto& indices) + // { + // cl[ilvl][indices].add_interval(interval); + // }); + // } - return Mesh_t(cl, minlevel, maxlevel); - } + // return Mesh_t(cl, minlevel, maxlevel); + // } - template - Mesh_t remove(Mesh_t& mesh, CellArray_t& lca) - { - using cl_type = typename Mesh_t::cl_type; + // template + // Mesh_t remove(Mesh_t& mesh, CellArray_t& lca) + // { + // using cl_type = typename Mesh_t::cl_type; - auto& refmesh = mesh[Mesh_t::mesh_id_t::cells]; + // auto& refmesh = mesh[Mesh_t::mesh_id_t::cells]; - auto minlevel = std::min(refmesh.min_level(), lca.min_level()); - auto maxlevel = std::max(refmesh.max_level(), lca.max_level()); + // auto minlevel = std::min(refmesh.min_level(), lca.min_level()); + // auto maxlevel = std::max(refmesh.max_level(), lca.max_level()); - // remove cells - cl_type cl; - size_t diff_ncells = 0; - for (size_t ilvl = minlevel; ilvl <= maxlevel; ++ilvl) - { - auto diff = samurai::difference(refmesh[ilvl], lca[ilvl]); + // // remove cells + // cl_type cl; + // size_t diff_ncells = 0; + // for (size_t ilvl = minlevel; ilvl <= maxlevel; ++ilvl) + // { + // auto diff = samurai::difference(refmesh[ilvl], lca[ilvl]); - diff( - [&](auto& interval, auto& index) - { - cl[ilvl][index].add_interval(interval); - diff_ncells += interval.size(); - }); - } + // diff( + // [&](auto& interval, auto& index) + // { + // cl[ilvl][index].add_interval(interval); + // diff_ncells += interval.size(); + // }); + // } - // new mesh for current process - return Mesh_t(cl, minlevel, maxlevel); - } + // // new mesh for current process + // return Mesh_t(cl, minlevel, maxlevel); + // } - } + // } struct MPI_Load_Balance { @@ -105,6 +107,12 @@ namespace samurai INTERVAL }; + enum Weight{ + OnSmall, + OnLarge, + None + }; + /** * Compute distance base on different norm. */ @@ -426,14 +434,14 @@ namespace samurai } template - void update_fields(Mesh_t& new_mesh) const + void update_fields([[maybe_unused]]Mesh_t& new_mesh) const { } public: - template - void load_balance(Mesh& mesh, Field_t& field, Fields&... kw) + template + void load_balance(Mesh_t & mesh, Field_t& field, Fields&... kw) { // specific load balancing strategy auto new_mesh = static_cast(this)->load_balance_impl(mesh); @@ -446,12 +454,16 @@ namespace samurai SAMURAI_TRACE("[LoadBalancer::load_balance]::Swapping meshes ... "); field.mesh().swap(new_mesh); - // discover neighbours: add new neighbours if a new interface appears or remove old neighbours - discover_neighbour< static_cast(Mesh::dim) >( new_mesh ); - discover_neighbour< static_cast(Mesh::dim) >( new_mesh ); + // discover neighbours: add new neighbours if a new interface appears or remove old neighbours + // FIX: add boolean return to condition the need of another call, might save some MPI comm. + discover_neighbour( new_mesh ); + discover_neighbour( new_mesh ); } /** + * This function MUST be used for debug or analysis purposes of load balancing strategy, + * it involves a lots of MPI communications. + * * Try to evaluate / compute a load balancing score. We expect from a good load * balancing strategy to: * - optimize the number of neighbours (reduce comm.) @@ -478,23 +490,23 @@ namespace samurai boost::mpi::all_gather(world, my_load_c, load_cells); } - if (world.rank() == 0) - { - std::cerr << "\t> LoadBalancer statistics : " << std::endl; + // if (world.rank() == 0) + // { + // std::cerr << "\t> LoadBalancer statistics : " << std::endl; - std::vector::iterator min_load = std::min_element(load_cells.begin(), load_cells.end()); - auto rank = std::distance(load_cells.begin(), min_load); - std::cerr << "\t\t> Min load {" << *min_load << ", cells } @ rank # " << rank << std::endl; + // std::vector::iterator min_load = std::min_element(load_cells.begin(), load_cells.end()); + // auto rank = std::distance(load_cells.begin(), min_load); + // std::cerr << "\t\t> Min load {" << *min_load << ", cells } @ rank # " << rank << std::endl; - std::vector::iterator max_load = std::max_element(load_cells.begin(), load_cells.end()); - rank = std::distance(load_cells.begin(), max_load); - std::cerr << "\t\t> Max load {" << *max_load << ", cells } @ rank # " << rank << std::endl; - } + // std::vector::iterator max_load = std::max_element(load_cells.begin(), load_cells.end()); + // rank = std::distance(load_cells.begin(), max_load); + // std::cerr << "\t\t> Max load {" << *max_load << ", cells } @ rank # " << rank << std::endl; + // } - std::string _stats = fmt::format("statistics_process_{}", world.rank()); - samurai::Statistics s(_stats); + // std::string _stats = fmt::format("statistics_process_{}", world.rank()); + // samurai::Statistics s(_stats); - s("stats", mesh); + // s("stats", mesh); // s( "statistics" , mesh ); @@ -507,13 +519,14 @@ namespace samurai /** * Precomputed direction to face-to-face element for both 3D and 2D */ - template + template constexpr auto getDirectionFace() { using base_stencil = xt::xtensor_fixed>; - xt::xtensor_fixed> stencils; - std::size_t nstencils = static_cast(2 * dim); + constexpr size_t size_ = 2 * dim; + xt::xtensor_fixed> stencils; + std::size_t nstencils = size_; for (std::size_t ist = 0; ist < nstencils; ++ist) { stencils[ist].fill(0); @@ -529,7 +542,7 @@ namespace samurai /** * Precomputed direction to diagonal element for both 3D and 2D cases. */ - template + template constexpr auto getDirectionDiag() { using base_stencil = xt::xtensor_fixed>; @@ -605,7 +618,7 @@ namespace samurai * by passing in parameters an array for example to modulate * weight according to level */ - template + template xt::xtensor_fixed> _cmpCellBarycenter(Mesh_t& mesh) { using Coord_t = xt::xtensor_fixed>; @@ -617,14 +630,15 @@ namespace samurai samurai::for_each_cell(mesh, [&](auto& cell) { - // all cells equals - // constexpr double wght = 1.; - - // higher weight for large cells - // double wght = 1. / static_cast( 1 << cell.level ); + double wght = 1.; + + if constexpr( w == Weight::OnSmall ){ + wght = 1. / static_cast( 1 << (mesh.max_level() - cell.level) ); + } - // higher weight for small cells - double wght = 1. / static_cast(1 << (mesh.max_level() - cell.level)); + if constexpr( w == Weight::OnLarge ){ + wght = 1. / static_cast( 1 << cell.level ); + } auto cc = cell.center(); @@ -645,12 +659,14 @@ namespace samurai return bary; } + + /** * * Params: * leveldiff : max level difference. For 2:1 balance, leveldiff = 1 */ - template + template static auto cmptInterface(Mesh_t& mesh, Mesh_t& omesh) { using CellList_t = typename Mesh_t::cl_type; @@ -929,10 +945,12 @@ namespace samurai * By default, stencils are "1" based, which means that we move the mesh by one unit element. This * could be changed if necessary by multiplying the stencils by an integer value. */ - template + template bool intersectionExists(Mesh_t& meshA, Mesh_t& meshB) { using mesh_id_t = typename Mesh_t::mesh_id_t; + + constexpr size_t dim = Mesh_t::dim; // operation are on leaves cells only auto meshA_ = meshA[mesh_id_t::cells]; @@ -982,7 +1000,7 @@ namespace samurai /** * Discover new neighbour connection that might arise during load balancing */ - template + template void discover_neighbour(Mesh_t& mesh) { using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; @@ -1007,7 +1025,7 @@ namespace samurai for (size_t nbi = 0; nbi < neighbourhood.size(); ++nbi) { // check current - neighbour connection - keepNeighbour[nbi] = intersectionExists(mesh, neighbourhood[nbi].mesh); + keepNeighbour[nbi] = intersectionExists(mesh, neighbourhood[nbi].mesh); // require update if a connection is lost // requireGeneralUpdate = requireGeneralUpdate || ( ! keepNeighbour[ nbi ] ); @@ -1022,7 +1040,7 @@ namespace samurai { logs << fmt::format("Checking neighbourhood connection {} <-> {}", neighbourhood[nbi].rank, neighbourhood[nbj].rank) << std::endl; - bool connected = intersectionExists(neighbourhood[nbi].mesh, neighbourhood[nbj].mesh); + bool connected = intersectionExists(neighbourhood[nbi].mesh, neighbourhood[nbj].mesh); if (connected) { @@ -1411,151 +1429,3 @@ namespace samurai // } // } - -// template -// void perform_load_balancing_diffusion( AMesh_t & global, int ndomains, const std::vector & all, Field_t & -// fake_mpi_rank ) { - -// using CellList_t = typename AMesh_t::cl_type; -// using cell_t = typename AMesh_t::cell_t; - -// struct Coord_t { xt::xtensor_fixed> coord; }; -// struct Logical_coord_t { xt::xtensor_fixed> coord; }; - -// std::vector barycenters ( ndomains ); - -// { // compute barycenter of current domains ( here, all domains since with simulate multiple MPI domains) - -// int maxlevel = 4; -// for( size_t m_=0; m_ < meshes.size(); ++m_ ) { - -// double wght_tot = 0.; -// samurai::for_each_cell( meshes[ m_ ], [&]( const auto & cell ) { - -// // [OPTIMIZATION] precompute weight as array -// double wght = 1. / ( 1 << ( maxlevel - cell.level ) ); - -// const auto cc = cell.center(); - -// barycenters[ m_ ].coord( 0 ) += cc( 0 ) * wght; -// barycenters[ m_ ].coord( 1 ) += cc( 1 ) * wght; -// if constexpr ( dim == 3 ) { barycenters[ m_ ].coord( 2 ) += cc( 2 ) * wght; } - -// wght_tot += wght; - -// }); - -// barycenters[ m_ ].coord( 0 ) /= wght_tot; -// barycenters[ m_ ].coord( 1 ) /= wght_tot; -// if constexpr ( dim == 3 ) barycenters[ m_ ].coord( 2 ) /= wght_tot; - -// std::cerr << "\t> Domain # " << m_ << ", bc : {" << barycenters[ m_ ].coord( 0 ) << ", " -// << barycenters[ m_ ].coord( 1 ) << "}" << std::endl; - -// } - -// } - -// std::vector new_meshes( ndomains ); -// std::vector exchanged( ndomains ); - -// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains - -// std::cerr << "\t> Working on domains # " << m_ << std::endl; - -// auto dist = samurai::make_field( "rank", meshes[ m_ ] ); - -// // neighbours offset to which data must be sent. -// // This is not equivalent to the MPI rank -// std::vector id_send; - -// int n_neighbours = static_cast( all[ m_ ].neighbour.size() ); -// for(std::size_t nbi = 0; nbi already_given( n_neighbours, 0 ); - -// // loop over each interval of the current ( or meshes[ m_ ] ) domain and check if it needs to be sent -// // to a given neighbour -// for_each_interval( meshes[ m_ ], [&]( std::size_t level, const auto& interval, const auto& index ){ -// Coord_t ibar; - -// std::cerr << "\t\t> Interval [" << interval.start << ", " << interval.end << "[" << std::endl; - -// double dm = 1. / (1 << level ); -// ibar.coord( 0 ) = ( (interval.end - interval.start) * 0.5 + interval.start ) * dm ; -// ibar.coord( 1 ) = ( index( 1 ) * dm ) + dm * 0.5; -// if constexpr ( dim == 3 ) ibar( 2 ) = ( index( 2 ) * dm ) + dm * 0.5; - -// std::cerr << "\t\t\t> level " << level << " center @ {" << ibar.coord(0) << ", " << ibar.coord(1) << "}" << std::endl; - -// // find which neighbour will potentially receive this interval -// int winner_id = -1; // keep it to current if still negative -// double winner_dist = std::numeric_limits::max(); -// for( int nbi=0; nbi Dist to neighbour #" << neighbour_rank << " : " << dist << " vs " << winner_dist << std::endl; -// std::cerr << "\t\t\t> Already given to this neighbour " << already_given[ neighbour_rank ] << std::endl; -// std::cerr << "\t\t\t> NbCells of this interval " << interval.size() << std::endl; -// std::cerr << "\t\t\t> fluxes for this neighbour : " << all[ m_ ].fluxes[ id_send[ nbi ] ] << std::endl; - -// if( dist < winner_dist && -// already_given[ neighbour_rank ] + interval.size() <= (- all[ m_ ].fluxes[ id_send[ nbi ] ]) ){ - -// winner_id = id_send[ nbi ]; -// winner_dist = dist; -// } - -// } - -// if( winner_id >= 0 ){ -// auto neighbour_rank = all[ m_ ].neighbour[ winner_id ]; -// std::cerr << "\t> Interval given to process " << neighbour_rank << " + ncells : " << interval.size() << std::endl; -// exchanged[ neighbour_rank ][ level ][ index ].add_interval( interval ); -// already_given[ neighbour_rank ] += interval.size(); -// }else{ -// new_meshes[ m_ ][ level ][ index ].add_interval( interval ); -// } - -// }); - -// meshes[ m_ ] = { new_meshes[ m_ ], true }; - -// } - -// for( std::size_t m_ = 0; m_ < meshes.size(); ++m_ ){ // over each domains -// for( int level=global.min_level(); level<=global.max_level(); ++level ) { -// auto intersect = intersection( global[ level ], meshes[ m_ ][ level ]); - -// intersect([&]( auto & i, auto & index ) { -// fake_mpi_rank( level, i, index ) = static_cast( m_ ); -// }); -// } - -// AMesh_t tmp = { exchanged[ m_], true }; -// for( int level=global.min_level(); level<=global.max_level(); ++level ) { -// auto intersect = intersection( global[ level ], tmp[ level ]); - -// intersect([&]( auto & i, auto & index ) { -// fake_mpi_rank( level, i, index ) = static_cast( m_ ); -// }); -// } -// } - -// } From 15681f5dc983fe38b792055fc254e68264356485 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 7 May 2024 22:24:03 +0200 Subject: [PATCH 080/170] update new load balancing strategy --- include/samurai/load_balancing_diffusion.hpp | 230 ++++++++----------- 1 file changed, 102 insertions(+), 128 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 158c93a52..224fb9d62 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -4,7 +4,10 @@ #include "load_balancing.hpp" #include -namespace load_balancing{ +// for std::sort +#include + +namespace Load_balancing{ class Diffusion : public samurai::LoadBalancer { @@ -12,6 +15,11 @@ namespace load_balancing{ int _ndomains; int _rank; + template + void propagate( const Mesh_t & mesh, const Stencil & dir, Field_t & field, int value, int &given ) const { + + } + public: Diffusion() { @@ -26,7 +34,7 @@ namespace load_balancing{ #endif } - inline std::string getName() const { return "diffusion"; } + inline std::string getName() const { return "diffusion"; } template Mesh_t load_balance_impl( Mesh_t & mesh ){ @@ -36,9 +44,8 @@ namespace load_balancing{ using CellArray_t = typename Mesh_t::ca_type; using mesh_id_t = typename Mesh_t::mesh_id_t; - using Coord_t = xt::xtensor_fixed>; - - constexpr int dim = static_cast( Mesh_t::dim ); + using Coord_t = xt::xtensor_fixed>; + using Stencil = xt::xtensor_fixed>; boost::mpi::communicator world; @@ -73,164 +80,131 @@ namespace load_balancing{ std::vector cl_to_send( n_neighbours ); + // set field "flags" for each rank. Initialized to current for all cells (leaves only) auto flags = samurai::make_field("diffusion_flag", mesh); for_each_interval( mesh[ mesh_id_t::cells ], [&]( std::size_t level, const auto & interval, const auto & index ){ flags( level, interval, index ) = _rank; }); - - - bool balancing_done = false; - while( ! balancing_done ){ - - // select neighbour with the highest needs of load - bool neighbour_found = false; - std::size_t requester = 0; - int requested_load = 0; - for(std::size_t nbi=0; nbi= 0 ) continue; - - // FIX: Add condition (&& interface not empty ) ? - // Neighbourhood should have been updated in a way that an interface exists ! - if( - new_fluxes[ nbi ] > requested_load ){ - requested_load = - new_fluxes[ nbi ]; - requester = nbi; - neighbour_found = true; - } - } + // load balancing order + std::vector order( n_neighbours ); + { + for( size_t i=0; i( mesh, neighbourhood[ requester ].mesh ); + // neighbour [0, n_neighbours[ + auto neighbour_local_id = order[ neigh_i ]; - { // check emptyness of interface, if it is empty, then set fluxes for this neighbour to 0 - size_t nelement = 0; - samurai::for_each_interval( interface, [&]( [[maybe_unused]] std::size_t level, [[maybe_unused]] const auto & interval, - [[maybe_unused]] const auto & index ){ - nelement += 1; - }); + // all cells have been given, neighbours that might left are "givers" (remember the fluxes were sorted) + if( fluxes[ neighbour_local_id ] >= 0 ) break; - if( nelement == 0 ){ - new_fluxes[ requester ] = 0; + // compute initial interface with this neighbour + auto interface = samurai::cmptInterface( mesh, neighbourhood[ neighbour_local_id ].mesh ); + samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ - { // debug - std::cerr << fmt::format("\t> Process {}, Warning no interface with a requester neighbour # {}", world.rank(), - neighbourhood[ requester ].rank ) << std::endl; - logs << "Requester neighbour, no interface found, set fluxes to " << new_fluxes[ requester ] << std::endl; - } - }else{ - { // debug - logs << "Requester neighbour, interface " << nelement << " intervals " << std::endl; - } - } - } + // this might be not correct. We might need a cell-basis loop + if( flags[ interval.start + interval.index ] == world.rank() ) + flags( level, interval, index ) = - neighbourhood[ neighbour_local_id ].rank; + }); - // go through interval on the interface and add as much as possible - // skip this to add the whole interface - CellList_t cl_for_neighbour; + // move the interface in the direction of "the center of mass" of the domain + // we basically want to move based on the normalized cartesian axis + // + // Q?: take into account already given intervals to compute BC ? + // (no weight here) + Coord_t bc_current = samurai::_cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t bc_neighbour = samurai::_cmpCellBarycenter( neighbourhood[ neighbour_local_id ].mesh[ mesh_id_t::cells ] ); + + // Compute normalized direction to neighbour, i.e. stencil + Stencil dir_from_neighbour; + { + Coord_t tmp; + double n2 = 0.; + for( size_t idim = 0; idim( tmp( idim ) / 0.5 ); } - }); + logs << fmt::format("\t\t> stencil for neighbour # {} :", neighbourhood[ neighbour_local_id ].rank); + for(size_t idim=0; idim Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; + + int nbInterGiven = 0; + CellList_t cl_given; - } + nbInterStep = 0; + for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Data transfer between processes ------------------------------------------------------------------ */ - /* ---------------------------------------------------------------------------------------------------------- */ - - CellList_t new_cl, need_remove; + // translate interface in direction of center of current mesh + auto set = samurai::translate( interface[ level ], dir_from_neighbour ); + auto intersect = samurai::intersection( set, mesh[ mesh_id_t::cells ][ level ]).on( level ); // need handle level difference here ! - // at this point local mesh of neighbour are modified ( technically it should match what would results from send) - // and local mesh has been modified - for(std::size_t ni=0; ni 0 ) { // receive data - CellArray_t to_rcv; - world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); + cl_given[ level ][ index ].add_interval( interval ); + } - logs << fmt::format("\t>[load_balance_impl]::interval Rank # {} receiving {} cells from rank # {}", world.rank(), to_rcv.nb_cells(), neighbourhood[ ni ].rank) << std::endl; - - // old strategy - // mesh.merge( to_rcv ); - - // add to future mesh of current process - samurai::for_each_interval(to_rcv, - [&](std::size_t level, const auto& interval, const auto& index) - { - new_cl[ level ][ index ].add_interval( interval ); }); - }else{ // send data to - CellArray_t to_send = { cl_to_send[ ni ], false }; - world.send( neighbourhood[ ni ].rank, 42, to_send ); + logs << fmt::format("\t\t\t> NbIntervalTot : {} vs NbIntervalGiven : {}", nbInterStep, nbInterGiven ) << std::endl; - logs << fmt::format("\t>[load_balance_impl]::interval Rank # {} sending {} cells to rank # {}", world.rank(), to_send.nb_cells(), neighbourhood[ ni ].rank) << std::endl; + } - samurai::for_each_interval( to_send, - [&](std::size_t level, const auto& interval, const auto& index) - { - need_remove[ level ][ index ].add_interval( interval ); - }); - } + interface = { cl_given, false }; - } + new_fluxes[ neighbour_local_id ] += nbInterGiven; - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Construct new mesh for current process ----------------------------------------------------------- */ - /* ---------------------------------------------------------------------------------------------------------- */ + logs << fmt::format("\t\t\t> Number of interval given to rank {} : {}", neighbourhood[ neighbour_local_id ].rank, nbInterGiven ) << std::endl; + } + + } + + if( new_fluxes[ neighbour_local_id ] < 0 ){ + std::cerr << "\t> Error cannot fullfill the neighbour ! " << std::endl; + } - // add to new_cl interval that were not sent - CellArray_t need_remove_ca = { need_remove }; // to optimize - for( size_t level=mesh.min_level(); level<=mesh.max_level(); ++level ){ - auto diff = samurai::difference( mesh[ mesh_id_t::cells ][ level ], need_remove_ca[ level ] ); + // only one neighbour processed + break; - diff([&]( auto & interval, auto & index ){ - new_cl[ level ][ index ].add_interval( interval ); - }); } + const std::string fn = fmt::format("test-diffusion"); + samurai::save( fn, mesh, flags ); + + CellList_t new_cl; Mesh_t new_mesh( new_cl, mesh ); return new_mesh; From 5d75d93724d5d9727df9b19674e2b5cce2fafdd4 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 7 May 2024 22:24:39 +0200 Subject: [PATCH 081/170] add new strategy for test in demo --- demos/FiniteVolume/advection_2d.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 57d68c970..15c556aaf 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -261,7 +261,7 @@ int main(int argc, char* argv[]) // SFC_LoadBalancer_interval balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; - load_balancing::Diffusion balancer; + Load_balancing::Diffusion balancer; while (t != Tf) { @@ -269,15 +269,9 @@ int main(int argc, char* argv[]) { std::cout << "\t> Load balancing mesh ... " << std::endl; - std::string suffix = fmt::format("_loadbalanced_bf_{}", nt); - save( path, filename, u, suffix); - myTimers.start("load-balancing"); balancer.load_balance(mesh, u); myTimers.stop("load-balancing"); - - suffix = fmt::format("_loadbalanced_af_{}", nt); - save( path, filename, u, suffix); } myTimers.start("MRadaptation"); From 8949be5ec89c8bfac9522ff378fe2e7c51689b06 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 7 May 2024 22:25:03 +0200 Subject: [PATCH 082/170] ifx missing libdeps in cmake --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3dc1117b4..1dc957071 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -239,7 +239,7 @@ endif() # Package the project package_project( - TARGETS samurai project_options project_warnings + TARGETS samurai project_options project_warnings libdeps INTERFACE_DEPENDENCIES_CONFIGURED ${DEPENDENCIES_CONFIGURED} INTERFACE_INCLUDES ${INCLUDE_DIR} ) From 576e0ec3b1ae9204d41c674b024ac35c26797375 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 14 May 2024 09:52:48 +0200 Subject: [PATCH 083/170] fix assert --- include/samurai/load_balancing.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index be9406e24..84a6b2696 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -241,9 +241,6 @@ namespace samurai my_load_new += transfertLoad; } - // Cannot have a negative or null load on an MPI process ! - assert(my_load_new > 0); - return fluxes; } From 2d5ea8841e001a13e446f5c31c9941f95de6f847 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 14 May 2024 09:53:20 +0200 Subject: [PATCH 084/170] load balacing interface propagation with local field --- include/samurai/load_balancing_diffusion.hpp | 156 ++++++++++++++----- 1 file changed, 119 insertions(+), 37 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 224fb9d62..734a065bf 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -55,12 +55,12 @@ namespace Load_balancing{ logs << fmt::format("> New load-balancing using {} ", getName() ) << std::endl; // compute fluxes in terms of number of intervals to transfer/receive - std::vector fluxes = samurai::cmptFluxes( mesh ); + std::vector fluxes = samurai::cmptFluxes( mesh ); std::vector new_fluxes( fluxes ); // get loads from everyone std::vector loads; - int my_load = static_cast( samurai::cmptLoad( mesh ) ); + int my_load = static_cast( samurai::cmptLoad( mesh ) ); boost::mpi::all_gather( world, my_load, loads ); std::vector & neighbourhood = mesh.mpi_neighbourhood(); @@ -82,9 +82,7 @@ namespace Load_balancing{ // set field "flags" for each rank. Initialized to current for all cells (leaves only) auto flags = samurai::make_field("diffusion_flag", mesh); - for_each_interval( mesh[ mesh_id_t::cells ], [&]( std::size_t level, const auto & interval, const auto & index ){ - flags( level, interval, index ) = _rank; - }); + flags.fill( world.rank() ); // load balancing order std::vector order( n_neighbours ); @@ -98,7 +96,7 @@ namespace Load_balancing{ } - for(size_t neigh_i=0; neigh_i= 0 ) break; + logs << fmt::format("\t> Working on neighbour # {}", neighbourhood[ neighbour_local_id ].rank ) << std::endl; + // compute initial interface with this neighbour auto interface = samurai::cmptInterface( mesh, neighbourhood[ neighbour_local_id ].mesh ); - samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ - // this might be not correct. We might need a cell-basis loop - if( flags[ interval.start + interval.index ] == world.rank() ) - flags( level, interval, index ) = - neighbourhood[ neighbour_local_id ].rank; - }); + { + size_t nCellsAtInterfaceGiven = 0, nCellsAtInterface = 0; + samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ + + for(size_t ii=0; ii NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}", nCellsAtInterface, nCellsAtInterfaceGiven ) << std::endl; + } // move the interface in the direction of "the center of mass" of the domain // we basically want to move based on the normalized cartesian axis @@ -140,7 +149,7 @@ namespace Load_balancing{ dir_from_neighbour( idim ) = static_cast( tmp( idim ) / 0.5 ); } - logs << fmt::format("\t\t> stencil for neighbour # {} :", neighbourhood[ neighbour_local_id ].rank); + logs << fmt::format("\t\t> stencil for this neighbour # {} :", neighbourhood[ neighbour_local_id ].rank); for(size_t idim=0; idim Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; - logs << fmt::format("\t\t\t> Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; + while( new_fluxes[ neighbour_local_id ] < 0 && nbInterStep != 0 ){ - int nbInterGiven = 0; + int nbGiven = 0; CellList_t cl_given; nbInterStep = 0; for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { - // translate interface in direction of center of current mesh - auto set = samurai::translate( interface[ level ], dir_from_neighbour ); - auto intersect = samurai::intersection( set, mesh[ mesh_id_t::cells ][ level ]).on( level ); // need handle level difference here ! + std::size_t minlevel_check = static_cast( std::max(static_cast(mesh.min_level()), static_cast(level) - 1 ) ); + std::size_t maxlevel_check = std::min( mesh.max_level(), level + 1 ); - intersect( [&]( const auto & interval, [[maybe_unused]] const auto & index ){ - - nbInterStep += 1; - // if( flags[ interval.start + interval.index ] == world.rank() ) - { - flags[ interval.start + interval.index ] = -10 * neighbourhood[ neighbour_local_id ].rank; - nbInterGiven += 1; + size_t nCellsAtInterface = 0, nCellsAtInterfaceGiven = 0; + for (size_t proj_level = minlevel_check; proj_level <= maxlevel_check; ++proj_level){ - cl_given[ level ][ index ].add_interval( interval ); + // translate interface in direction of center of current mesh + auto set = samurai::translate( interface[ level ], dir_from_neighbour ); + auto intersect = samurai::intersection( set, mesh[ mesh_id_t::cells ][ proj_level ]).on( proj_level ); // need handle level difference here ! + + if ( proj_level > level) { + auto set_ = samurai::translate( interface[ proj_level ], dir_from_neighbour ); + auto diff_ = samurai::difference( intersect, set_ ); + + diff_( [&]( const auto & interval, const auto & index ) { + for(size_t ii=0; ii NbIntervalTot : {} vs NbIntervalGiven : {}", nbInterStep, nbInterGiven ) << std::endl; + logs << fmt::format("\t\t\t\t> At level {}, NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}, (offset:{})", level, + nCellsAtInterface, nCellsAtInterfaceGiven, offset ) << std::endl; } interface = { cl_given, false }; - new_fluxes[ neighbour_local_id ] += nbInterGiven; - - logs << fmt::format("\t\t\t> Number of interval given to rank {} : {}", neighbourhood[ neighbour_local_id ].rank, nbInterGiven ) << std::endl; + new_fluxes[ neighbour_local_id ] += nbGiven; } } if( new_fluxes[ neighbour_local_id ] < 0 ){ - std::cerr << "\t> Error cannot fullfill the neighbour ! " << std::endl; + std::cerr << fmt::format("\t> Error cannot fullfill the neighbour # {} ", neighbourhood[ neighbour_local_id ].rank ) << std::endl; } - // only one neighbour processed - break; + } + + CellList_t new_cl; + std::vector payload( world.size() ); + + samurai::for_each_cell( mesh[mesh_id_t::cells], [&]( const auto & cell ){ + + if( flags[ cell ] == world.rank() ){ + if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + }else{ + if constexpr ( Mesh_t::dim == 1 ){ payload[ flags[ cell ] ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + } + + }); + + // const std::string fn = fmt::format("test-diffusion"); + // samurai::save( fn, mesh, flags ); + + /* ---------------------------------------------------------------------------------------------------------- */ + /* ------- Data transfer between processes ------------------------------------------------------------------ */ + /* ---------------------------------------------------------------------------------------------------------- */ + + for( size_t neigh_i=0; neigh_i 0 ){ // receiver + CellArray_t to_rcv; + world.recv( neighbourhood[ neigh_i ].rank, 42, to_rcv ); + + samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ + new_cl[ level ][ index ].add_interval( interval ); + }); + + }else{ // sender + CellArray_t to_send = { payload[ neighbourhood[ neigh_i ].rank ], false }; + world.send( neighbourhood[ neigh_i ].rank, 42, to_send ); + + } } - const std::string fn = fmt::format("test-diffusion"); - samurai::save( fn, mesh, flags ); + /* ---------------------------------------------------------------------------------------------------------- */ + /* ------- Construct new mesh for current process ----------------------------------------------------------- */ + /* ---------------------------------------------------------------------------------------------------------- */ - CellList_t new_cl; Mesh_t new_mesh( new_cl, mesh ); return new_mesh; From 9ebb472d106b9418677084d013344930690927db Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 14 May 2024 14:34:47 +0200 Subject: [PATCH 085/170] fix diffusion load balancing with field by removing diagonal exchanges --- include/samurai/load_balancing_diffusion.hpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 734a065bf..1ebe6ec32 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -156,6 +156,21 @@ namespace Load_balancing{ logs << std::endl; } + // Avoid diagonals exchange, and emphaze x-axis + + if constexpr ( Mesh_t::dim == 2 ) { + if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 ){ + dir_from_neighbour[1] = 0; + } + } + + if constexpr ( Mesh_t::dim == 3 ) { + if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 && std::abs(dir_from_neighbour[2]) == 1){ + dir_from_neighbour[1] = 0; + dir_from_neighbour[2] = 0; + } + } + // propagate in direction { int nbInterStep = 1; // validate the while condition on starter @@ -256,9 +271,6 @@ namespace Load_balancing{ }); - // const std::string fn = fmt::format("test-diffusion"); - // samurai::save( fn, mesh, flags ); - /* ---------------------------------------------------------------------------------------------------------- */ /* ------- Data transfer between processes ------------------------------------------------------------------ */ /* ---------------------------------------------------------------------------------------------------------- */ From 9b5998d03cbd441b3f4c0ee9cbc54ebd5fe7a8b3 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 16 May 2024 09:29:54 +0200 Subject: [PATCH 086/170] load balancing thresold + cmptInterfaceUniform --- include/samurai/load_balancing.hpp | 101 +++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 84a6b2696..158fbb0ed 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -113,6 +113,8 @@ namespace samurai None }; + static const double load_balancing_threshold = 0.025; // 2.5 % + /** * Compute distance base on different norm. */ @@ -241,6 +243,22 @@ namespace samurai my_load_new += transfertLoad; } + // apply threshold, if the difference is smaller than #load_balancing_threshold of the number of cells, + // we do not load balance those processes + for (std::size_t n_i = 0; n_i < n_neighbours; ++n_i) + { + std::size_t neighbour_rank = static_cast( neighbourhood[ n_i ].rank ); + int neighbour_load = loads[neighbour_rank]; + int abs_diff = std::abs( fluxes[ n_i ] ); + int threshold_neigh = static_cast( load_balancing_threshold * loads[ neighbour_rank ] ); + int threshold_curr = static_cast( load_balancing_threshold * my_load ); + + if( abs_diff < threshold_curr && abs_diff < threshold_neigh ){ + fluxes[ n_i ] = 0; + } + + } + return fluxes; } @@ -741,6 +759,89 @@ namespace samurai return interface_; } + /** + * + * Params: + * leveldiff : max level difference. For 2:1 balance, leveldiff = 1 + */ + template + static auto cmptInterfaceUniform(Mesh_t& mesh, Mesh_t& omesh, const Dir_t & dir ) + { + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; + using mesh_id_t = typename Mesh_t::mesh_id_t; + + // operation are on leaves only + auto& currentMesh = mesh[mesh_id_t::cells]; + auto& otherMesh = omesh[mesh_id_t::cells]; + + size_t minlevel = otherMesh.min_level(); + size_t maxlevel = otherMesh.max_level(); + + size_t minLevelAtInterface = 99; + + struct MinMax { + int min_x = std::numeric_limits::max(); + int max_x = std::numeric_limits::min(); + int min_y = std::numeric_limits::max(); + int max_y = std::numeric_limits::min(); + }; + + std::vector< MinMax > mm ( std::max( mesh.max_level(), omesh.max_level() ) + 1 ); + + for (size_t level = minlevel; level <= maxlevel; ++level) + { + // for each level we need to check level -1 / 0 / +1 + std::size_t minlevel_check = static_cast( + std::max(static_cast(currentMesh.min_level()), static_cast(level - 1) )); + std::size_t maxlevel_check = std::min(currentMesh.max_level(), level + 1); + + for (size_t projlevel = minlevel_check; projlevel <= maxlevel_check; ++projlevel) + { + // translate neighbour from dir (hopefully to current) all direction are tested + auto set = translate( otherMesh[level], dir ); + auto intersect = intersection(set, currentMesh[projlevel]).on(projlevel); + + size_t nbInter_ = 0; + intersect( [&]( const auto & interval, const auto & index ) { + nbInter_ += 1; + + mm[ projlevel ].min_x = std::min( interval.start, mm[ projlevel ].min_x ); + mm[ projlevel ].max_x = std::max( interval.end, mm[ projlevel ].max_x ); + mm[ projlevel ].min_y = std::min( index(0), mm[ projlevel ].min_y ); + mm[ projlevel ].max_y = std::max( index(0), mm[ projlevel ].max_y ); + }); + + if (nbInter_ > 0){ + std::cerr << " not empty " << std::endl; + minLevelAtInterface = std::min( projlevel, minLevelAtInterface ); + } + } + + } + + CellList_t tmp; + tmp[ minLevelAtInterface ][ { mm[ minLevelAtInterface ].min_y } ].add_interval( { mm[ minLevelAtInterface ].min_x, mm[ minLevelAtInterface ].max_x } ); + CellArray_t ca_tmp = { tmp, false }; + + CellList_t cl_interface; + + size_t nbInter_ = 0; + for (size_t level=currentMesh.min_level(); level <= currentMesh.max_level(); ++level) + { + auto intersect = intersection( ca_tmp[ minLevelAtInterface ], currentMesh[ level ] ).on( level ); + + intersect( [&](const auto & interval, const auto & index) { + cl_interface[ level ][ index ].add_interval( interval ); + nbInter_ += 1; + }); + } + + CellArray_t interface = { cl_interface, false }; + + return interface; + } + /** * Compute cells at the interface between geometricaly adjacent * domains. It relies on neighbourhood mpi_subdomain_t data structure From 29c85011d8d9b697e828a9661341fa693975d130 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 16 May 2024 09:30:06 +0200 Subject: [PATCH 087/170] update interface prop strategy --- include/samurai/load_balancing_diffusion.hpp | 76 ++++++++++++++------ 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 1ebe6ec32..2218d9d3a 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -96,6 +96,8 @@ namespace Load_balancing{ } + samurai::save("./", "lb-diffusion-before", mesh, flags ); + for( size_t neigh_i=0; neigh_i Working on neighbour # {}", neighbourhood[ neighbour_local_id ].rank ) << std::endl; - // compute initial interface with this neighbour - auto interface = samurai::cmptInterface( mesh, neighbourhood[ neighbour_local_id ].mesh ); - - { - size_t nCellsAtInterfaceGiven = 0, nCellsAtInterface = 0; - samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ - - for(size_t ii=0; ii NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}", nCellsAtInterface, nCellsAtInterfaceGiven ) << std::endl; - } - // move the interface in the direction of "the center of mass" of the domain // we basically want to move based on the normalized cartesian axis // @@ -171,6 +155,27 @@ namespace Load_balancing{ } } + // compute initial interface with this neighbour + // using Dir_t = xt::xtensor_fixed>; + // Dir_t xx = { 0, -1 }; + + auto interface = samurai::cmptInterfaceUniform( mesh, neighbourhood[ neighbour_local_id ].mesh, dir_from_neighbour ); + + { + size_t nCellsAtInterfaceGiven = 0, nCellsAtInterface = 0; + samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ + + for(size_t ii=0; ii NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}", nCellsAtInterface, nCellsAtInterfaceGiven ) << std::endl; + } + // propagate in direction { int nbInterStep = 1; // validate the while condition on starter @@ -181,13 +186,36 @@ namespace Load_balancing{ logs << fmt::format("\t\t\t> Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; while( new_fluxes[ neighbour_local_id ] < 0 && nbInterStep != 0 ){ + + size_t minLevelInInterface = mesh.max_level(); + for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { + std::size_t minlevel_check = static_cast( std::max(static_cast(mesh.min_level()), static_cast(level) - 1 ) ); + std::size_t maxlevel_check = std::min( mesh.max_level(), level + 1 ); + size_t nIntervalAtInterface = 0; + for (size_t proj_level = minlevel_check; proj_level <= maxlevel_check; ++proj_level){ + + // translate interface in direction of center of current mesh + auto set = samurai::translate( interface[ level ], dir_from_neighbour ); + auto intersect = samurai::intersection( set, mesh[ mesh_id_t::cells ][ proj_level ]).on( proj_level ); // need handle level difference here ! + intersect( [&]( [[maybe_unused]] const auto & interval, [[maybe_unused]] const auto & index ){ + nIntervalAtInterface += 1; + }); + + if( nIntervalAtInterface > 0 ) minLevelInInterface = std::min( minLevelInInterface, proj_level ); + } + } + logs << fmt::format("\t\t\t\t> Min level in interface : {}", minLevelInInterface ) << std::endl; + int nbGiven = 0; CellList_t cl_given; nbInterStep = 0; for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { + // int factor_diff_level = std::max( level - minLevelInInterface, static_cast( 1 ) ); + int factor_diff_level = 1 ; + std::size_t minlevel_check = static_cast( std::max(static_cast(mesh.min_level()), static_cast(level) - 1 ) ); std::size_t maxlevel_check = std::min( mesh.max_level(), level + 1 ); @@ -195,11 +223,11 @@ namespace Load_balancing{ for (size_t proj_level = minlevel_check; proj_level <= maxlevel_check; ++proj_level){ // translate interface in direction of center of current mesh - auto set = samurai::translate( interface[ level ], dir_from_neighbour ); + auto set = samurai::translate( interface[ level ], dir_from_neighbour * factor_diff_level ); auto intersect = samurai::intersection( set, mesh[ mesh_id_t::cells ][ proj_level ]).on( proj_level ); // need handle level difference here ! if ( proj_level > level) { - auto set_ = samurai::translate( interface[ proj_level ], dir_from_neighbour ); + auto set_ = samurai::translate( interface[ proj_level ], dir_from_neighbour * factor_diff_level ); auto diff_ = samurai::difference( intersect, set_ ); diff_( [&]( const auto & interval, const auto & index ) { @@ -214,7 +242,9 @@ namespace Load_balancing{ } }); - }else{ + } + else + { intersect( [&]( const auto & interval, [[maybe_unused]] const auto & index ){ nbInterStep += 1; @@ -244,6 +274,8 @@ namespace Load_balancing{ interface = { cl_given, false }; new_fluxes[ neighbour_local_id ] += nbGiven; + + nbInterStep = 0; } } @@ -254,6 +286,8 @@ namespace Load_balancing{ } + samurai::save("./", "lb-diffusion", mesh, flags); + CellList_t new_cl; std::vector payload( world.size() ); From 49199419e83611a5b13a632caf8acbe241406204 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 16 May 2024 10:50:58 +0200 Subject: [PATCH 088/170] fix global min/max for interface uniform --- include/samurai/load_balancing.hpp | 32 +++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 158fbb0ed..860aedb5e 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -794,11 +794,11 @@ namespace samurai // for each level we need to check level -1 / 0 / +1 std::size_t minlevel_check = static_cast( std::max(static_cast(currentMesh.min_level()), static_cast(level - 1) )); - std::size_t maxlevel_check = std::min(currentMesh.max_level(), level + 1); + std::size_t maxlevel_check = std::min(currentMesh.max_level(), level + static_cast( 1 ) ); for (size_t projlevel = minlevel_check; projlevel <= maxlevel_check; ++projlevel) { - // translate neighbour from dir (hopefully to current) all direction are tested + // translate neighbour from dir (hopefully to current) auto set = translate( otherMesh[level], dir ); auto intersect = intersection(set, currentMesh[projlevel]).on(projlevel); @@ -820,8 +820,34 @@ namespace samurai } + // boost::mpi::communicator world; + // std::ofstream logs; + // logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + // logs << fmt::format( "\t\t\t> Interface min, max : x ({},{}); min, max : y ({},{}) @ level : {}", mm[ minLevelAtInterface ].min_x, + // mm[ minLevelAtInterface ].max_x, mm[ minLevelAtInterface ].min_y, mm[ minLevelAtInterface ].max_y, minLevelAtInterface ) << std::endl; + + struct MinMax global; + global.min_x = mm[ minLevelAtInterface ].min_x; + global.max_x = mm[ minLevelAtInterface ].max_x; + global.min_y = mm[ minLevelAtInterface ].min_y; + global.max_y = mm[ minLevelAtInterface ].max_y; + + for( size_t level=minLevelAtInterface+1; level At level {}, diff {}", level, diff_level ) << std::endl; + // logs << fmt::format( "\t\t\t> Origin min, max : x ({},{}); min, max : y ({},{})", mm[ level ].min_x, + // mm[ level ].max_x, mm[ level ].min_y, mm[ level ].max_y ) << std::endl; + // logs << fmt::format( "\t\t\t> Eq minlevel, min, max : x ({},{}); min, max : y ({},{}) ", mm[ level ].min_x >> diff_level, + // mm[ level ].max_x >> diff_level, mm[ level ].min_y >> diff_level, mm[ level ].max_y >> diff_level ) << std::endl; + + global.min_x = std::min( global.min_x, mm[ level ].min_x >> diff_level ); + global.max_x = std::max( global.max_x, mm[ level ].max_x >> diff_level ); + global.min_y = std::min( global.min_y, mm[ level ].min_y >> diff_level ); + global.max_y = std::max( global.max_y, mm[ level ].max_y >> diff_level ); + } + CellList_t tmp; - tmp[ minLevelAtInterface ][ { mm[ minLevelAtInterface ].min_y } ].add_interval( { mm[ minLevelAtInterface ].min_x, mm[ minLevelAtInterface ].max_x } ); + tmp[ minLevelAtInterface ][ { global.min_y } ].add_interval( { global.min_x, global.max_x } ); CellArray_t ca_tmp = { tmp, false }; CellList_t cl_interface; From 87b3dae8a682b668fa35bd8d1b5700b168d00877 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 17 May 2024 15:44:12 +0200 Subject: [PATCH 089/170] fix load balancing base interface --- include/samurai/load_balancing.hpp | 59 +++++++++++++++++------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 860aedb5e..0af174db8 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -113,7 +113,7 @@ namespace samurai None }; - static const double load_balancing_threshold = 0.025; // 2.5 % + static const double load_balancing_threshold = 0.03141592; // 2.5 % /** * Compute distance base on different norm. @@ -471,8 +471,8 @@ namespace samurai // discover neighbours: add new neighbours if a new interface appears or remove old neighbours // FIX: add boolean return to condition the need of another call, might save some MPI comm. - discover_neighbour( new_mesh ); - discover_neighbour( new_mesh ); + discover_neighbour( field.mesh() ); + discover_neighbour( field.mesh() ); } /** @@ -761,12 +761,14 @@ namespace samurai /** * - * Params: - * leveldiff : max level difference. For 2:1 balance, leveldiff = 1 */ template static auto cmptInterfaceUniform(Mesh_t& mesh, Mesh_t& omesh, const Dir_t & dir ) { + + // dim == 3 is not supported for the moment. + assert( dim == 2 ); + using CellList_t = typename Mesh_t::cl_type; using CellArray_t = typename Mesh_t::ca_type; using mesh_id_t = typename Mesh_t::mesh_id_t; @@ -787,7 +789,7 @@ namespace samurai int max_y = std::numeric_limits::min(); }; - std::vector< MinMax > mm ( std::max( mesh.max_level(), omesh.max_level() ) + 1 ); + std::vector< MinMax > mm ( std::max( mesh.max_level(), omesh.max_level() ) + 2 ); for (size_t level = minlevel; level <= maxlevel; ++level) { @@ -820,9 +822,9 @@ namespace samurai } - // boost::mpi::communicator world; - // std::ofstream logs; - // logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + boost::mpi::communicator world; + std::ofstream logs; + logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); // logs << fmt::format( "\t\t\t> Interface min, max : x ({},{}); min, max : y ({},{}) @ level : {}", mm[ minLevelAtInterface ].min_x, // mm[ minLevelAtInterface ].max_x, mm[ minLevelAtInterface ].min_y, mm[ minLevelAtInterface ].max_y, minLevelAtInterface ) << std::endl; @@ -847,25 +849,33 @@ namespace samurai } CellList_t tmp; - tmp[ minLevelAtInterface ][ { global.min_y } ].add_interval( { global.min_x, global.max_x } ); - CellArray_t ca_tmp = { tmp, false }; + + // +y, -y : horizontal propagation + if( std::abs( dir[0] ) == 0 && std::abs( dir[1] ) == 1 ){ + logs << "\t> Horizontal propagation ! " << std::endl; + if( dir[ 1 ] > 0 ){ + tmp[ minLevelAtInterface ][ { global.min_y } ].add_interval( { global.min_x, global.max_x } ); + } else { + tmp[ minLevelAtInterface ][ { global.max_y } ].add_interval( { global.min_x, global.max_x } ); + } + } - CellList_t cl_interface; + // +x, -x : vertical propagation + if( std::abs( dir[0] ) == 1 && std::abs( dir[1] ) == 0 ){ + logs << "\t> Vertical propagation ! " << std::endl; + for(int y=global.min_y; y<=global.max_y; ++y ){ + tmp[ minLevelAtInterface ][ { y } ].add_interval( { global.min_x, global.max_x } ); + } + } - size_t nbInter_ = 0; - for (size_t level=currentMesh.min_level(); level <= currentMesh.max_level(); ++level) - { - auto intersect = intersection( ca_tmp[ minLevelAtInterface ], currentMesh[ level ] ).on( level ); - intersect( [&](const auto & interval, const auto & index) { - cl_interface[ level ][ index ].add_interval( interval ); - nbInter_ += 1; - }); - } - - CellArray_t interface = { cl_interface, false }; + CellArray_t ca_tmp = { tmp, false }; + + logs << "Interface for propagation : " << std::endl; + logs << ca_tmp << std::endl; + logs << std::endl; - return interface; + return ca_tmp; } /** @@ -1238,7 +1248,6 @@ namespace samurai // gather neighbour mesh mesh.update_mesh_neighbour(); - // return requireGeneralUpdate; } } // namespace samurai From 4abe9989e56b6221454f700de7a18793c53276dd Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 17 May 2024 15:44:24 +0200 Subject: [PATCH 090/170] fix laod blancing diffusion --- include/samurai/load_balancing_diffusion.hpp | 191 +++++++++---------- 1 file changed, 91 insertions(+), 100 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 2218d9d3a..84c01456d 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -116,7 +116,7 @@ namespace Load_balancing{ Coord_t bc_current = samurai::_cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); Coord_t bc_neighbour = samurai::_cmpCellBarycenter( neighbourhood[ neighbour_local_id ].mesh[ mesh_id_t::cells ] ); - // Compute normalized direction to neighbour, i.e. stencil + // Compute normalized direction to neighbour Stencil dir_from_neighbour; { Coord_t tmp; @@ -132,156 +132,147 @@ namespace Load_balancing{ tmp( idim ) /= n2; dir_from_neighbour( idim ) = static_cast( tmp( idim ) / 0.5 ); } + + // Avoid diagonals exchange, and emphaze x-axis. Maybe two phases propagation in case of diagonal ? + // i.e.: if (1, 1) -> (1, 0) then (0, 1) ? + + if constexpr ( Mesh_t::dim == 2 ) { + if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 ){ + dir_from_neighbour[0] = 0; + dir_from_neighbour[1] = 0; + } + } + + if constexpr ( Mesh_t::dim == 3 ) { + if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 && std::abs(dir_from_neighbour[2]) == 1){ + dir_from_neighbour[1] = 0; + dir_from_neighbour[2] = 0; + } + } - logs << fmt::format("\t\t> stencil for this neighbour # {} :", neighbourhood[ neighbour_local_id ].rank); + logs << fmt::format("\t\t> (corrected) stencil for this neighbour # {} :", neighbourhood[ neighbour_local_id ].rank); for(size_t idim=0; idim( mesh, neighbourhood[ neighbour_local_id ].mesh, dir_from_neighbour ); - if constexpr ( Mesh_t::dim == 2 ) { - if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 ){ - dir_from_neighbour[1] = 0; - } + bool empty = false; + { + size_t iii = 0; + samurai::for_each_interval( interface, [&](const auto & level, const auto & i, const auto ii ){ + iii ++; + }); + if( iii == 0 ) empty = true; } - if constexpr ( Mesh_t::dim == 3 ) { - if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 && std::abs(dir_from_neighbour[2]) == 1){ - dir_from_neighbour[1] = 0; - dir_from_neighbour[2] = 0; - } + if( empty ) { + logs << "\t> Skipping neighbour, empty interface ! " << std::endl; + continue; } - // compute initial interface with this neighbour - // using Dir_t = xt::xtensor_fixed>; - // Dir_t xx = { 0, -1 }; - - auto interface = samurai::cmptInterfaceUniform( mesh, neighbourhood[ neighbour_local_id ].mesh, dir_from_neighbour ); - { size_t nCellsAtInterfaceGiven = 0, nCellsAtInterface = 0; - samurai::for_each_interval( interface, [&]( std::size_t level, const auto & interval, const auto & index ){ - - for(size_t ii=0; ii NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}", nCellsAtInterface, nCellsAtInterfaceGiven ) << std::endl; } - // propagate in direction + // propagate until fullfill neighbour { - int nbInterStep = 1; // validate the while condition on starter - - // let's suppose the interface is up-to-date based on "flags" already given - int offset = 1; - + int nbElementGiven = 1; // validate the while condition on starter + logs << fmt::format("\t\t\t> Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; - while( new_fluxes[ neighbour_local_id ] < 0 && nbInterStep != 0 ){ + int offset = 1; + while( new_fluxes[ neighbour_local_id ] < 0 && nbElementGiven > 0 ){ + // intersection of interface with current mesh size_t minLevelInInterface = mesh.max_level(); - for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { - std::size_t minlevel_check = static_cast( std::max(static_cast(mesh.min_level()), static_cast(level) - 1 ) ); - std::size_t maxlevel_check = std::min( mesh.max_level(), level + 1 ); - size_t nIntervalAtInterface = 0; - for (size_t proj_level = minlevel_check; proj_level <= maxlevel_check; ++proj_level){ - - // translate interface in direction of center of current mesh - auto set = samurai::translate( interface[ level ], dir_from_neighbour ); - auto intersect = samurai::intersection( set, mesh[ mesh_id_t::cells ][ proj_level ]).on( proj_level ); // need handle level difference here ! + + { + auto interface_on_mesh = samurai::translate( interface[ interface.min_level() ], dir_from_neighbour * offset ); // interface is monolevel ! + for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { + size_t nIntervalAtInterface = 0; + auto intersect = samurai::intersection( interface_on_mesh, mesh[ mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! intersect( [&]( [[maybe_unused]] const auto & interval, [[maybe_unused]] const auto & index ){ nIntervalAtInterface += 1; }); - if( nIntervalAtInterface > 0 ) minLevelInInterface = std::min( minLevelInInterface, proj_level ); + if( nIntervalAtInterface > 0 ) minLevelInInterface = std::min( minLevelInInterface, level ); } } + + if( minLevelInInterface != interface.min_level() ) { + logs << "\t\t\t\t> [WARNING] Interface need to be update !" << std::endl; + } logs << fmt::format("\t\t\t\t> Min level in interface : {}", minLevelInInterface ) << std::endl; - int nbGiven = 0; - CellList_t cl_given; + // CellList_t cl_given; - nbInterStep = 0; - for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { + nbElementGiven = 0; - // int factor_diff_level = std::max( level - minLevelInInterface, static_cast( 1 ) ); - int factor_diff_level = 1 ; - - std::size_t minlevel_check = static_cast( std::max(static_cast(mesh.min_level()), static_cast(level) - 1 ) ); - std::size_t maxlevel_check = std::min( mesh.max_level(), level + 1 ); - - size_t nCellsAtInterface = 0, nCellsAtInterfaceGiven = 0; - for (size_t proj_level = minlevel_check; proj_level <= maxlevel_check; ++proj_level){ - - // translate interface in direction of center of current mesh - auto set = samurai::translate( interface[ level ], dir_from_neighbour * factor_diff_level ); - auto intersect = samurai::intersection( set, mesh[ mesh_id_t::cells ][ proj_level ]).on( proj_level ); // need handle level difference here ! - - if ( proj_level > level) { - auto set_ = samurai::translate( interface[ proj_level ], dir_from_neighbour * factor_diff_level ); - auto diff_ = samurai::difference( intersect, set_ ); - - diff_( [&]( const auto & interval, const auto & index ) { - for(size_t ii=0; ii At level {}, NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}, nbElementGiven : {}", level, + nCellsAtInterface, nCellsAtInterfaceGiven, nbElementGiven ) << std::endl; - logs << fmt::format("\t\t\t\t> At level {}, NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}, (offset:{})", level, - nCellsAtInterface, nCellsAtInterfaceGiven, offset ) << std::endl; + nbElementGiven += nCellsAtInterfaceGiven; } - interface = { cl_given, false }; - - new_fluxes[ neighbour_local_id ] += nbGiven; + // interface = { cl_given, false }; - nbInterStep = 0; + new_fluxes[ neighbour_local_id ] += nbElementGiven; + offset ++; } } if( new_fluxes[ neighbour_local_id ] < 0 ){ - std::cerr << fmt::format("\t> Error cannot fullfill the neighbour # {} ", neighbourhood[ neighbour_local_id ].rank ) << std::endl; + logs << fmt::format("\t> Error cannot fullfill the neighbour # {}, fluxes: {} ", neighbourhood[ neighbour_local_id ].rank, new_fluxes[ neighbour_local_id ] ) << std::endl; } } From 89e71338586a84fce43b9fc446e4e4e40691f60c Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 17 May 2024 15:44:44 +0200 Subject: [PATCH 091/170] update advection 2d test --- demos/FiniteVolume/advection_2d.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 15c556aaf..7f276638f 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -209,7 +209,7 @@ int main(int argc, char* argv[]) // Output parameters fs::path path = fs::current_path(); std::string filename = "FV_advection_2d"; - std::size_t nfiles = 1; + std::size_t nfiles = 10; int nt_loadbalance = 10; // nombre d'iteratio entre les equilibrages app.add_option("--min-corner", min_corner, "The min corner of the box")->capture_default_str()->group("Simulation parameters"); @@ -257,17 +257,25 @@ int main(int argc, char* argv[]) std::size_t nsave = 1; std::size_t nt = 0; + + const int iof = 2; // SFC_LoadBalancer_interval balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; Load_balancing::Diffusion balancer; + std::ofstream logs; + boost::mpi::communicator world; + logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + while (t != Tf) { if (nt % nt_loadbalance == 0 && nt > 1 ) { - std::cout << "\t> Load balancing mesh ... " << std::endl; + logs << fmt::format("\n##########################################", nt ) << std::endl; + logs << fmt::format("\n> Load balancing mesh @ iteration {} ", nt ) << std::endl; + logs << fmt::format("\n##########################################", nt ) << std::endl; myTimers.start("load-balancing"); balancer.load_balance(mesh, u); @@ -305,7 +313,7 @@ int main(int argc, char* argv[]) std::swap(u.array(), unp1.array()); myTimers.start("I/O"); - if (t >= static_cast(nsave + 1) * dt_save || t == Tf || true ) + if (t >= static_cast(nsave + 1) * dt_save || t == Tf || nt % iof == 0 ) { const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; save(path, filename, u, suffix); From f01d2cabb09b8dcea87437b825cd9b8d59d0ec83 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 21 May 2024 16:45:20 +0200 Subject: [PATCH 092/170] fix load balancing crash --- include/samurai/load_balancing.hpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 0af174db8..66db41b4c 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -789,7 +789,16 @@ namespace samurai int max_y = std::numeric_limits::min(); }; - std::vector< MinMax > mm ( std::max( mesh.max_level(), omesh.max_level() ) + 2 ); + boost::mpi::communicator world; + std::ofstream logs; + logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + + logs << fmt::format("\t\t\t> Allocating vector of size : max({}, {})+2 : {}", mesh.max_level(), omesh.max_level(), + std::max( mesh.max_level(), omesh.max_level() ) + 2 ) << std::endl; + + // FIXME: omesh.max_level() sometimes equals to uintmax .. why ?? + size_t msize = std::min( std::max( mesh.max_level(), omesh.max_level() ) + 2, static_cast( 22 ) ); + std::vector< MinMax > mm ( msize ); for (size_t level = minlevel; level <= maxlevel; ++level) { @@ -822,11 +831,9 @@ namespace samurai } - boost::mpi::communicator world; - std::ofstream logs; - logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); - // logs << fmt::format( "\t\t\t> Interface min, max : x ({},{}); min, max : y ({},{}) @ level : {}", mm[ minLevelAtInterface ].min_x, - // mm[ minLevelAtInterface ].max_x, mm[ minLevelAtInterface ].min_y, mm[ minLevelAtInterface ].max_y, minLevelAtInterface ) << std::endl; + logs << fmt::format("\t\t\t> minLevelAtInterface : {}", minLevelAtInterface ) << std::endl; + logs << fmt::format( "\t\t\t> Interface min, max : x ({},{}); min, max : y ({},{}) @ level : {}", mm[ minLevelAtInterface ].min_x, + mm[ minLevelAtInterface ].max_x, mm[ minLevelAtInterface ].min_y, mm[ minLevelAtInterface ].max_y, minLevelAtInterface ) << std::endl; struct MinMax global; global.min_x = mm[ minLevelAtInterface ].min_x; From a50eb0ef346c058f5f7885184ac4c569254f282d Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 22 May 2024 11:28:24 +0200 Subject: [PATCH 093/170] FIX bug max level for archive in serizalize --- include/samurai/mesh.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 4426b19f9..3c4706cb2 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -202,7 +202,7 @@ namespace samurai ar & m_subdomain; ar & m_union; ar & m_min_level; - ar & m_min_level; + ar & m_max_level; } #endif }; From 0e100d0b818404135f9f9038a4e05c7764b4f4b0 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 22 May 2024 11:28:43 +0200 Subject: [PATCH 094/170] remove fix crash --- include/samurai/load_balancing.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 66db41b4c..a0513df9e 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -797,7 +797,8 @@ namespace samurai std::max( mesh.max_level(), omesh.max_level() ) + 2 ) << std::endl; // FIXME: omesh.max_level() sometimes equals to uintmax .. why ?? - size_t msize = std::min( std::max( mesh.max_level(), omesh.max_level() ) + 2, static_cast( 22 ) ); + // size_t msize = std::min( std::max( mesh.max_level(), omesh.max_level() ) + 2, static_cast( 22 ) ); + size_t msize = std::max( mesh.max_level(), omesh.max_level() ) + 2; std::vector< MinMax > mm ( msize ); for (size_t level = minlevel; level <= maxlevel; ++level) From c10b5468fabe240dfff4719bbcb721e8765db90f Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 29 May 2024 15:59:14 +0200 Subject: [PATCH 095/170] remove useless constructor --- include/samurai/mr/mesh.hpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/include/samurai/mr/mesh.hpp b/include/samurai/mr/mesh.hpp index 67c9b4d47..54a6d633f 100644 --- a/include/samurai/mr/mesh.hpp +++ b/include/samurai/mr/mesh.hpp @@ -86,13 +86,13 @@ namespace samurai const std::array& periodic, double approx_box_tol = lca_type::default_approx_box_tol, double scaling_factor = 0); - // Used for load balancing - MRMesh(const cl_type & cl, - std::size_t min_level, - std::size_t max_level, - std::vector & neighbourhood, - double approx_box_tol = lca_type::default_approx_box_tol, - double scaling_factor = 0); + // Used for load balancing, finally commented + //MRMesh(const cl_type & cl, + // std::size_t min_level, + // std::size_t max_level, + // std::vector & neighbourhood, + // double approx_box_tol = lca_type::default_approx_box_tol, + // double scaling_factor = 0); void update_sub_mesh_impl(); @@ -139,12 +139,12 @@ namespace samurai { } - template - inline MRMesh::MRMesh( const cl_type & cl, std::size_t min_level, std::size_t max_level, - std::vector & neighbourhood) - : base_type(cl, min_level, max_level, neighbourhood ) - { - } + // template + // inline MRMesh::MRMesh( const cl_type & cl, std::size_t min_level, std::size_t max_level, + // std::vector & neighbourhood) + // : base_type(cl, min_level, max_level, neighbourhood ) + // { + // } template inline void MRMesh::update_sub_mesh_impl() From 7d3458fac456e49500fcde45bc5697ef76ee08db Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 29 May 2024 16:00:22 +0200 Subject: [PATCH 096/170] update load balancer with reordering + update field using all to all meshes --- include/samurai/load_balancing.hpp | 246 +++++++++++++++++++++++++---- 1 file changed, 211 insertions(+), 35 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index a0513df9e..a9569fcce 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -113,7 +113,7 @@ namespace samurai None }; - static const double load_balancing_threshold = 0.03141592; // 2.5 % + static const double load_balancing_threshold = 0.00; // 0.03141592; // 2.5 % /** * Compute distance base on different norm. @@ -198,12 +198,109 @@ namespace samurai * */ template - std::vector cmptFluxes(Mesh_t& mesh) + std::vector cmptFluxes( Mesh_t& mesh, const std::vector & neighbourhood, int niterations ) + { + + boost::mpi::communicator world; + + std::ofstream logs; + logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + + size_t n_neighbours = neighbourhood.size(); + + // load of current process + int my_load = static_cast( cmptLoad( mesh ) ); + + // fluxes between processes + std::vector fluxes(n_neighbours, 0); + + // load of each process (all processes not only neighbours) + std::vector loads; + + // numbers of neighbours processes for each process, used for weighting fluxes + std::vector neighbourhood_n_neighbours; + + // number of neighbours for each process + boost::mpi::all_gather(world, neighbourhood.size(), neighbourhood_n_neighbours); + + // get "my_load" from other processes + int nt = 0; + while( nt < niterations ){ + boost::mpi::all_gather(world, my_load, loads); + + // compute updated my_load for current process based on its neighbourhood + int my_load_new = my_load; + for (std::size_t n_i = 0; n_i < n_neighbours; ++n_i) + { + std::size_t neighbour_rank = static_cast( neighbourhood[ n_i ] ); + int neighbour_load = loads[ neighbour_rank ]; + double diff_load = static_cast( neighbour_load - my_load_new); + + std::size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[ neighbour_rank ]; + + double weight = 1. / static_cast( std::max( n_neighbours, nb_neighbours_neighbour ) + 1 ); + + // if transferLoad < 0 -> need to send data, if transferLoad > 0 need to receive data + int transfertLoad = static_cast( std::lround( weight * diff_load )) ; + + fluxes[ n_i ] += transfertLoad; + + // my_load_new += transfertLoad; + + my_load += transfertLoad; + } + + logs << fmt::format("it {}, neighbours : ", nt) ; + for( size_t in=0; in( neighbourhood[ n_i ] ); + int neighbour_load = loads[neighbour_rank]; + int abs_diff = std::abs( fluxes[ n_i ] ); + int threshold_neigh = static_cast( load_balancing_threshold * loads[ neighbour_rank ] ); + int threshold_curr = static_cast( load_balancing_threshold * my_load ); + + if( abs_diff < threshold_curr && abs_diff < threshold_neigh ){ + fluxes[ n_i ] = 0; + } + + } + + return fluxes; + } + + /** + * Compute fluxes based on load computing stategy based on graph with label + * propagation algorithm. Return, for the current process, the flux in term of + * load, i.e. the quantity of "load" to transfer to its neighbours. If the load + * is negative, it means that the process (current) must send load to neighbour, + * if positive it means that it must receive load. + * + * This function use 2 MPI all_gather calls. + * + */ + template + std::vector cmptFluxes(Mesh_t& mesh, int niterations) { using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; boost::mpi::communicator world; + std::ofstream logs; + logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + // give access to geometricaly neighbour process rank and mesh std::vector& neighbourhood = mesh.mpi_neighbourhood(); size_t n_neighbours = neighbourhood.size(); @@ -220,27 +317,46 @@ namespace samurai // numbers of neighbours processes for each process, used for weighting fluxes std::vector neighbourhood_n_neighbours; - // get "my_load" from other processes - boost::mpi::all_gather(world, my_load, loads); + // number of neighbours for each process boost::mpi::all_gather(world, neighbourhood.size(), neighbourhood_n_neighbours); - // compute updated my_load for current process based on its neighbourhood - int my_load_new = my_load; - for (std::size_t n_i = 0; n_i < n_neighbours; ++n_i) - { - std::size_t neighbour_rank = static_cast(neighbourhood[n_i].rank); - int neighbour_load = loads[neighbour_rank]; - double diff_load = static_cast(neighbour_load - my_load); + // get "my_load" from other processes + int nt = 0; + while( nt < niterations ){ + boost::mpi::all_gather(world, my_load, loads); + + // compute updated my_load for current process based on its neighbourhood + int my_load_new = my_load; + for (std::size_t n_i = 0; n_i < n_neighbours; ++n_i) + { + std::size_t neighbour_rank = static_cast(neighbourhood[n_i].rank); + int neighbour_load = loads[neighbour_rank]; + double diff_load = static_cast( neighbour_load - my_load_new); - std::size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[neighbour_rank]; + std::size_t nb_neighbours_neighbour = neighbourhood_n_neighbours[neighbour_rank]; - double weight = 1. / static_cast(std::max(n_neighbours, nb_neighbours_neighbour) + 1); + double weight = 1. / static_cast(std::max(n_neighbours, nb_neighbours_neighbour) + 1); - int transfertLoad = static_cast(std::lround(weight * diff_load)); + // if transferLoad < 0 -> need to send data, if transferLoad > 0 need to receive data + int transfertLoad = static_cast(std::lround(weight * diff_load)); - fluxes[n_i] += transfertLoad; + fluxes[n_i] += transfertLoad; - my_load_new += transfertLoad; + // my_load_new += transfertLoad; + + my_load += transfertLoad; + } + + logs << fmt::format("it {}, neighbours : ", nt) ; + for( size_t in=0; in void update_field(Mesh_t& new_mesh, Field_t& field) const @@ -347,14 +464,24 @@ namespace samurai logs << fmt::format("> [LoadBalancer]::update_field rank # {}: data copied for intersection old/new ", world.rank() ) << std::endl; std::vector req; - std::vector> to_send(new_mesh.mpi_neighbourhood().size()); + std::vector> to_send(world.size()); std::size_t i_neigh = 0; + // FIXME: this is overkill and will not scale + std::vector all_new_meshes, all_old_meshes; + boost::mpi::all_gather( world, new_mesh, all_new_meshes ); + boost::mpi::all_gather( world, field.mesh(), all_old_meshes ); + // build payload of field that has been sent to neighbour, so compare old mesh with new neighbour mesh - for (auto& neighbour : new_mesh.mpi_neighbourhood()) + // for (auto& neighbour : new_mesh.mpi_neighbourhood()) + for( size_t ni=0; ni [LoadBalancer]::update_field rank # {}: data to send to {}", world.rank(), neighbour.rank ) << std::endl; + logs << fmt::format("> [LoadBalancer]::update_field rank # {}: data to send to {}", world.rank(), neighbour_rank ) << std::endl; } } logs << fmt::format("> [LoadBalancer]::update_field rank # {}, nb req isend {}", world.rank(), req.size() ) << std::endl; // build payload of field that I need to receive from neighbour, so compare NEW mesh with OLD neighbour mesh - for (auto& old_neighbour : old_mesh.mpi_neighbourhood()) + for (size_t ni=0; ni to_recv; std::ptrdiff_t count = 0; - auto in_interface = intersection(old_neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + auto in_interface = intersection( all_old_meshes[ ni ][mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); in_interface( [&]( [[maybe_unused]]const auto& i, [[maybe_unused]]const auto& index) @@ -409,13 +540,13 @@ namespace samurai { std::ptrdiff_t count = 0; std::vector to_recv; - world.recv(old_neighbour.rank, world.rank(), to_recv); + world.recv( ni, world.rank(), to_recv); for (std::size_t level = min_level; level <= max_level; ++level) { - if (!new_mesh[mesh_id_t::cells][level].empty() && !old_neighbour.mesh[mesh_id_t::cells][level].empty()) + if (!new_mesh[mesh_id_t::cells][level].empty() && !all_old_meshes[ ni ][mesh_id_t::cells][level].empty()) { - auto in_interface = intersection(old_neighbour.mesh[mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); + auto in_interface = intersection(all_old_meshes[ ni ][mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); in_interface( [&](const auto& i, const auto& index) @@ -455,24 +586,63 @@ namespace samurai public: + LoadBalancer() { + nloadbalancing = 0; + } + + /** + * This function reorder cells across MPI processes based on a + * Space Filling Curve. This is mandatory for load balancing using SFC. + */ + template + void reordering( Mesh_t & mesh, Field_t& field, Fields&... kw ) + { + // new reordered mesh on current process + MPI exchange with others + auto new_mesh = static_cast(this)->reordering_impl( mesh ); + + // update each physical field on the new reordered mesh + SAMURAI_TRACE("[Reordering::load_balance]::Updating fields ... "); + update_fields( new_mesh, field, kw... ); + + // swap mesh reference. + // FIX: this is not clean + SAMURAI_TRACE("[Reordering::load_balance]::Swapping meshes ... "); + field.mesh().swap( new_mesh ); + + // discover neighbours: add new neighbours if a new interface appears or remove old neighbours + discover_neighbour( field.mesh() ); + discover_neighbour( field.mesh() ); + + } + template void load_balance(Mesh_t & mesh, Field_t& field, Fields&... kw) { + + std::string lbn = static_cast(this)->getName(); + + auto p = lbn.find( "SFC" ); + if( nloadbalancing == 0 && p != std::string::npos ) { + reordering( mesh , field, kw... ); + } + // specific load balancing strategy - auto new_mesh = static_cast(this)->load_balance_impl(mesh); + auto new_mesh = static_cast(this)->load_balance_impl( field.mesh() ); // update each physical field on the new load balanced mesh SAMURAI_TRACE("[LoadBalancer::load_balance]::Updating fields ... "); - update_fields(new_mesh, field, kw...); + update_fields( new_mesh, field, kw... ); // swap mesh reference to new load balanced mesh. FIX: this is not clean SAMURAI_TRACE("[LoadBalancer::load_balance]::Swapping meshes ... "); - field.mesh().swap(new_mesh); + field.mesh().swap( new_mesh ); // discover neighbours: add new neighbours if a new interface appears or remove old neighbours // FIX: add boolean return to condition the need of another call, might save some MPI comm. discover_neighbour( field.mesh() ); discover_neighbour( field.mesh() ); + + nloadbalancing += 1; } /** @@ -796,8 +966,6 @@ namespace samurai logs << fmt::format("\t\t\t> Allocating vector of size : max({}, {})+2 : {}", mesh.max_level(), omesh.max_level(), std::max( mesh.max_level(), omesh.max_level() ) + 2 ) << std::endl; - // FIXME: omesh.max_level() sometimes equals to uintmax .. why ?? - // size_t msize = std::min( std::max( mesh.max_level(), omesh.max_level() ) + 2, static_cast( 22 ) ); size_t msize = std::max( mesh.max_level(), omesh.max_level() ) + 2; std::vector< MinMax > mm ( msize ); @@ -832,6 +1000,14 @@ namespace samurai } + if( minLevelAtInterface == 99 ) { + logs << "\t\t\t> No interface found ... " << std::endl; + + CellList_t tmp; + CellArray_t ca_tmp = { tmp, false }; + return ca_tmp; + } + logs << fmt::format("\t\t\t> minLevelAtInterface : {}", minLevelAtInterface ) << std::endl; logs << fmt::format( "\t\t\t> Interface min, max : x ({},{}); min, max : y ({},{}) @ level : {}", mm[ minLevelAtInterface ].min_x, mm[ minLevelAtInterface ].max_x, mm[ minLevelAtInterface ].min_y, mm[ minLevelAtInterface ].max_y, minLevelAtInterface ) << std::endl; From 40a32070a584c1c562bc46ebb81c3e19c3a4de36 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 29 May 2024 16:01:34 +0200 Subject: [PATCH 097/170] try fix interface prop lb --- include/samurai/load_balancing_diffusion.hpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 84c01456d..c67465dca 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -96,8 +96,6 @@ namespace Load_balancing{ } - samurai::save("./", "lb-diffusion-before", mesh, flags ); - for( size_t neigh_i=0; neigh_i( tmp( idim ) / 0.5 ); + + // FIXME why needed ? + if( std::abs( dir_from_neighbour( idim ) ) > 1 ) { + dir_from_neighbour( idim ) < 0 ? dir_from_neighbour( idim ) = -1 : dir_from_neighbour( idim ) = 1; + } } // Avoid diagonals exchange, and emphaze x-axis. Maybe two phases propagation in case of diagonal ? @@ -200,7 +203,7 @@ namespace Load_balancing{ logs << fmt::format("\t\t> NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}", nCellsAtInterface, nCellsAtInterfaceGiven ) << std::endl; } - // propagate until fullfill neighbour + // propagate until full-fill neighbour { int nbElementGiven = 1; // validate the while condition on starter @@ -277,8 +280,6 @@ namespace Load_balancing{ } - samurai::save("./", "lb-diffusion", mesh, flags); - CellList_t new_cl; std::vector payload( world.size() ); From 2735358f43e713be57affa5586770fa4e79a138e Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 29 May 2024 16:01:52 +0200 Subject: [PATCH 098/170] update SFC load balancing --- include/samurai/load_balancing_sfc.hpp | 455 +++++++++++++++++++++++-- 1 file changed, 419 insertions(+), 36 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 995919f05..38beeb0f9 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -13,7 +13,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer + Mesh_t reordering_impl( Mesh_t & mesh ) { + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = samurai::CellArray; + + boost::mpi::communicator world; + + // For debug + std::ofstream logs; + logs.open("log_" + std::to_string(_rank) + ".dat", std::ofstream::app); + logs << "# New load balancing (load_balancing_sfc)" << std::endl; + + // SFC 1D key for cells + auto sfc_keys = samurai::make_field( "keys", mesh ); + sfc_keys.fill( 0 ); + + auto flags = samurai::make_field("rank", mesh); + flags.fill( world.rank() ); + + logs << fmt::format("\t\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; + + SFC_key_t mink = std::numeric_limits::max(), maxk = std::numeric_limits::min(); + + size_t ncells = 0; + samurai::for_each_cell(mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ) { + + // this is where things can get nasty, we expect indices to be positive values !! + xt::xtensor_fixed> ijk; + for (size_t idim = 0; idim < dim; ++idim) { + + // FIX need shift to get only positive index + assert( cell.indices( 0 ) >= 0 ); + ijk( idim ) = static_cast( cell.indices( idim ) ) << ( mesh.max_level() - cell.level ); + } + + auto key = _sfc.template getKey( ijk ); + + sfc_keys[ cell ] = key; + + mink = std::min( key, mink ); + maxk = std::max( key, maxk ); + + ncells ++; + + }); + + // Key boundaries of current process - unused for now + std::vector bounds = { mink, maxk }; + logs << "\t\t\t> Local key bounds [" << bounds[ 0 ] << ", " << bounds[ 1 ] << "]" << std::endl; + + std::vector boundaries; + boost::mpi::all_gather( world, bounds.data(), bounds.size(), boundaries ); + + logs << "\t\t\t> Global key boundaries ["; + for(const auto & ik : boundaries ) + logs << ik << ","; + logs << "]" << std::endl; + + // Check overlap with previous/next process. Does not mean that there is no overlap, but at least between "adjacent" + // (MPI-1), (MPI+1) there is not overlap found + std::vector boundaries_new( world.size() + 1 ); + + // find max value for boundaries + SFC_key_t globalMax = boundaries[ 0 ]; + for(size_t ip=0; ip Global key evenly distrib boundaries ["; + for(const auto & ik : boundaries_new ) + logs << ik << ","; + logs << "]" << std::endl; + + // distribute cell based on boundaries & sfc key + std::map comm; + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&](const auto & cell ) { + auto key = sfc_keys[ cell ]; + + // optimize using bisect - find proc that should have this cell + for( size_t ip=0; ip= boundaries_new[ ip ] && key < boundaries_new[ ip + 1 ] ){ + flags[ cell ] = static_cast( ip ); + + // unique list of process that should be contacted + if( comm.find( static_cast( ip ) ) == comm.end() ){ + comm[ static_cast( ip ) ] = true; + } + + break; + } + } + + }); + + logs << "\t\t\t> Comm required with processes : ["; + for( const auto & it : comm ) + logs << it.first << ","; + logs << "]" << std::endl; + + // /* ---------------------------------------------------------------------------------------------------------- */ + // /* ------- Data transfer between processes ------------------------------------------------------------------ */ + // /* ---------------------------------------------------------------------------------------------------------- */ + + CellList_t new_cl; + std::vector payload( world.size() ); + + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ + + if( flags[ cell ] == world.rank() ){ + if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + }else{ + if constexpr ( Mesh_t::dim == 1 ){ payload[ flags[ cell ] ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + } + + }); + + // FIXME: this part involve a lot of communication since each process will communicate with all processes. + // This should be improved for better scalability. + for( int iproc=0; iproc + Mesh_t load_balance_impl( Mesh_t & mesh ) + { + using inter_t = samurai::Interval; + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = samurai::CellArray; + + boost::mpi::communicator world; + + // For debug + std::ofstream logs; + logs.open("log_" + std::to_string(_rank) + ".dat", std::ofstream::app); + logs << "# New load balancing (load_balancing_sfc)" << std::endl; + + // SFC 1D key for cells + std::map sfc_map; + auto sfc_keys = samurai::make_field( "keys", mesh ); + sfc_keys.fill( 0 ); + + auto flags = samurai::make_field("rank", mesh); + flags.fill( world.rank() ); + + logs << fmt::format("\t\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; + + SFC_key_t mink = std::numeric_limits::max(), maxk = std::numeric_limits::min(); + size_t ncells = 0; + samurai::for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto & cell ) { + + // this is where things can get nasty, we expect indices to be positive values !! + xt::xtensor_fixed> ijk; + for (size_t idim = 0; idim < dim; ++idim) { + + // FIX need shift to get only positive index + assert( cell.indices( 0 ) >= 0 ); + ijk( idim ) = static_cast( cell.indices( idim ) ) << ( mesh.max_level() - cell.level ); + } + + auto key = _sfc.template getKey( ijk ); + + sfc_keys[ cell ] = key; + + if( sfc_map.find( key ) != sfc_map.end() ) { + assert( false ); + std::cerr << fmt::format("Rank # {}, Error computing SFC, index not uniq ! ", world.rank()) << std::endl; + } + + sfc_map[ key ] = cell; + + mink = std::min( mink, key ); + maxk = std::max( maxk, key ); + + ncells ++; + + }); + + assert( ncells == sfc_map.size() ); + + size_t ncells_tot = 0, dc = 0; + std::vector nbCellsPerProc; + std::vector globIdx( world.size() + 1, 0 ), globIdxNew( world.size() + 1, 0 ); + boost::mpi::all_gather( world, mesh.nb_cells( Mesh_t::mesh_id_t::cells ), nbCellsPerProc ); + + logs << "\t\t\t> Number of cells : " << mesh.nb_cells( Mesh_t::mesh_id_t::cells ) << std::endl; + + for(size_t i=0; i GlobalIdx : "; + for(const auto & i : globIdx ) + logs << i << ", "; + logs << std::endl; + + logs << "\t\t\t> GlobalIdx balanced : "; + for(const auto & i : globIdxNew ) + logs << i << ", "; + logs << std::endl; + + } + + size_t start = 0; + + // for( size_t ip=0; ip globIdx[ world.rank() ] ) break; + // start ++; + // } + // start = std::min( start, static_cast( world.size() - 1 ) ); + while( globIdx[ world.rank() ] >= ( start + 1 ) * dc ){ + start ++; + } + + logs << "Start @ rank " << start << std::endl; + + size_t count = globIdx[ world.rank() ]; + for( auto & it : sfc_map ) { + + if( count >= ( start + 1 ) * dc ){ + start ++; + logs << "Incrementing Start @ rank " << start << ", count " << count << std::endl; + } + + flags[ it.second ] = start; + + count ++; + } + + // /* ---------------------------------------------------------------------------------------------------------- */ + // /* ------- Data transfer between processes ------------------------------------------------------------------ */ + // /* ---------------------------------------------------------------------------------------------------------- */ + + // distribute cell based on boundaries & sfc key + std::map comm; + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&](const auto & cell ) { + + if( comm.find( flags[ cell ] ) == comm.end() ) { + comm[ flags[ cell ] ] = true; + } + + }); + + logs << "\t\t\t> Comm required with processes : ["; + for( const auto & it : comm ) + logs << it.first << ","; + logs << "]" << std::endl; + + // /* ---------------------------------------------------------------------------------------------------------- */ + // /* ------- Data transfer between processes ------------------------------------------------------------------ */ + // /* ---------------------------------------------------------------------------------------------------------- */ + + CellList_t new_cl; + std::vector payload( world.size() ); + + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ + + if( flags[ cell ] == world.rank() ){ + if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + }else{ + if constexpr ( Mesh_t::dim == 1 ){ payload[ flags[ cell ] ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + } + + }); + + for( int iproc=0; iproc send {} to # {}", reqExchg, iproc ) << std::endl; + world.send( iproc, 17, reqExchg ); + + if( reqExchg == 1 ) { + CellArray_t to_send = { payload[ iproc ], false }; + world.send( iproc, 17, to_send ); + } + + } + + for( int iproc=0; iproc recv {} to # {}", reqExchg, iproc ) << std::endl; + + if( reqExchg == 1 ) { + CellArray_t to_rcv; + world.recv( iproc, 17, to_rcv ); + + samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ + new_cl[ level ][ index ].add_interval( interval ); + }); + } + + } + + + // /* ---------------------------------------------------------------------------------------------------------- */ + // /* ------- Construct new mesh for current process ----------------------------------------------------------- */ + // /* ---------------------------------------------------------------------------------------------------------- */ + + Mesh_t new_mesh( new_cl, mesh ); + + return new_mesh; + } + template - Mesh_t load_balance_impl( Mesh_t& mesh ) + Mesh_t load_balance_impl_old( Mesh_t& mesh ) { using inter_t = samurai::Interval; using CellList_t = typename Mesh_t::cl_type; @@ -56,49 +439,49 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer sfc_map; + + logs << fmt::format("\t\t> Computing SFC ({}) 1D indices (interval) ... ", _sfc.getName() ) << std::endl; size_t ninterval = 0; - samurai::for_each_interval(mesh, - [&](std::size_t level, const auto& inter, const auto& index) - { - // get Logical coordinate or first cell - xt::xtensor_fixed> icell; - - // first element of interval - icell(0) = inter.start; - for (int idim = 0; idim < dim - 1; ++idim) - { - icell(idim + 1) = index(idim); - } - - // convert logical coordinate to max level logical coordinates - for (int idim = 0; idim < dim; ++idim) - { - icell(idim) = icell(idim) << (mesh.max_level() - level ); // +1 - } - - // this is where things can get nasty, we expect indices to be positive values !! - xt::xtensor_fixed> ijk; - for (size_t idim = 0; idim < dim; ++idim) - { - ijk(idim) = static_cast(icell(idim)); - } - - sfc_map[_sfc.template getKey(ijk)] = {level, inter, index, false}; - - ninterval++; - }); + samurai::for_each_interval(mesh, [&]( std::size_t level, const auto & inter, const auto & index ) { + // get Logical coordinate or first cell + xt::xtensor_fixed> icell; + + // first element of interval + icell(0) = inter.start; + for (int idim = 0; idim < dim - 1; ++idim) + { + icell(idim + 1) = index(idim); + } + + // convert logical coordinate to max level logical coordinates + for (int idim = 0; idim < dim; ++idim) + { + icell(idim) = icell(idim) << (mesh.max_level() - level ); // +1 + } + + // this is where things can get nasty, we expect indices to be positive values !! + xt::xtensor_fixed> ijk; + for (size_t idim = 0; idim < dim; ++idim) + { + ijk(idim) = static_cast( icell(idim) ); + } + + sfc_map[_sfc.template getKey(ijk)] = {level, inter, index, false}; + + ninterval++; + }); assert(ninterval == sfc_map.size()); // Key boundaries of current process - unused for now - // SFC_key_t interval[ 2 ] = { sfc_map.begin()->first, sfc_map.rbegin()->first }; - // logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; + SFC_key_t interval[ 2 ] = { sfc_map.begin()->first, sfc_map.rbegin()->first }; + logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; std::vector load_interval; - int my_load_i = static_cast(samurai::cmptLoad(mesh)); + int my_load_i = static_cast(samurai::cmptLoad(mesh)); boost::mpi::all_gather(world, my_load_i, load_interval); // compute load to transfer to neighbour rank-1, rank+1 From 02300cc22bebb341df9fdadfda099aca39e107c2 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 30 May 2024 13:27:36 +0200 Subject: [PATCH 099/170] fix load balancing with SFC --- demos/FiniteVolume/advection_2d.cpp | 62 +++++++++++++++++++++----- include/samurai/load_balancing.hpp | 38 +++++++++------- include/samurai/load_balancing_sfc.hpp | 39 +++++++--------- 3 files changed, 89 insertions(+), 50 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 7f276638f..4610e7b05 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -31,6 +31,8 @@ auto init(Mesh& mesh, const double radius, const double x_center, const double y { auto u = samurai::make_field("u", mesh); + u.fill( 0. ); + samurai::for_each_cell( mesh, [&](auto& cell) @@ -41,12 +43,37 @@ auto init(Mesh& mesh, const double radius, const double x_center, const double y // const double y_center = 0.3; if (((center[0] - x_center) * (center[0] - x_center) + (center[1] - y_center) * (center[1] - y_center)) <= radius * radius) { - u[cell] = 1; + u[cell] += 1; } - else - { - u[cell] = 0; + + }); + + return u; +} + +template +auto init(Mesh& mesh, const std::vector & radius, const std::vector & xcenters, const std::vector & ycenters ) +{ + auto u = samurai::make_field("u", mesh); + + u.fill( 0. ); + + samurai::for_each_cell( + mesh, + [&](auto& cell) + { + auto center = cell.center(); + + for(std::size_t nc=0; nc(nfiles); double t = 0.; - auto u = init(mesh, radius, x_center, y_center); + std::vector xcenters, ycenters, radii; + xcenters.emplace_back( x_center ); + ycenters.emplace_back( y_center ); + radii.emplace_back( radius ); + + // xcenters = { 0.3, 1., 2.3 }; + // ycenters = { 0.3, 1.4, 3. }; + // radii = { 0.2, 0.23, 0.3 }; + xcenters = { 1., 3., 1. }; + ycenters = { 1., 1., 3. }; + radii = { 0.2, 0.2, 0.2 }; + + auto u = init(mesh, radii, xcenters, ycenters); + samurai::make_bc>(u, 0.); auto unp1 = samurai::make_field("unp1", mesh); @@ -258,12 +298,13 @@ int main(int argc, char* argv[]) std::size_t nsave = 1; std::size_t nt = 0; - const int iof = 2; + const int iof = 1; + + SFC_LoadBalancer_interval balancer; - // SFC_LoadBalancer_interval balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; - Load_balancing::Diffusion balancer; + // Load_balancing::Diffusion balancer; std::ofstream logs; boost::mpi::communicator world; @@ -273,10 +314,6 @@ int main(int argc, char* argv[]) { if (nt % nt_loadbalance == 0 && nt > 1 ) { - logs << fmt::format("\n##########################################", nt ) << std::endl; - logs << fmt::format("\n> Load balancing mesh @ iteration {} ", nt ) << std::endl; - logs << fmt::format("\n##########################################", nt ) << std::endl; - myTimers.start("load-balancing"); balancer.load_balance(mesh, u); myTimers.stop("load-balancing"); @@ -319,6 +356,7 @@ int main(int argc, char* argv[]) save(path, filename, u, suffix); } myTimers.stop("I/O"); + } myTimers.print(); diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index a9569fcce..f96cfc950 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -113,7 +113,7 @@ namespace samurai None }; - static const double load_balancing_threshold = 0.00; // 0.03141592; // 2.5 % + static const double load_balancing_threshold = 0.03141592; // 2.5 % /** * Compute distance base on different norm. @@ -430,18 +430,17 @@ namespace samurai class LoadBalancer { private: + std::ofstream logs; int nloadbalancing; template - void update_field(Mesh_t& new_mesh, Field_t& field) const + void update_field(Mesh_t& new_mesh, Field_t& field) { using mesh_id_t = typename Mesh_t::mesh_id_t; using value_t = typename Field_t::value_type; boost::mpi::communicator world; - std::ofstream logs; - logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); - logs << fmt::format("> [LoadBalancer]::update_field rank # {} -> '{}' ", world.rank(), field.name() ) << std::endl; + logs << fmt::format("\n# [LoadBalancer]::update_field rank # {} -> '{}' ", world.rank(), field.name() ) << std::endl; Field_t new_field("new_f", new_mesh); new_field.fill(0); @@ -461,8 +460,6 @@ namespace samurai intersect_old_new.apply_op( samurai::copy( new_field, field ) ); } - logs << fmt::format("> [LoadBalancer]::update_field rank # {}: data copied for intersection old/new ", world.rank() ) << std::endl; - std::vector req; std::vector> to_send(world.size()); @@ -502,11 +499,11 @@ namespace samurai req.push_back( world.isend( neighbour_rank, neighbour_rank, to_send[ ni ] ) ); // i_neigh ++; - logs << fmt::format("> [LoadBalancer]::update_field rank # {}: data to send to {}", world.rank(), neighbour_rank ) << std::endl; + logs << fmt::format("\t> [LoadBalancer]::update_field send data to rank # {}", neighbour_rank ) << std::endl; } } - logs << fmt::format("> [LoadBalancer]::update_field rank # {}, nb req isend {}", world.rank(), req.size() ) << std::endl; + logs << fmt::format("\t> [LoadBalancer]::update_field number of isend request: {}", req.size() ) << std::endl; // build payload of field that I need to receive from neighbour, so compare NEW mesh with OLD neighbour mesh for (size_t ni=0; ni - void update_fields(Mesh_t& new_mesh, Field_t& field, Fields_t&... kw) const + void update_fields(Mesh_t& new_mesh, Field_t& field, Fields_t&... kw) { update_field(new_mesh, field); @@ -580,16 +577,22 @@ namespace samurai } template - void update_fields([[maybe_unused]]Mesh_t& new_mesh) const + void update_fields([[maybe_unused]]Mesh_t& new_mesh) { } public: LoadBalancer() { + boost::mpi::communicator world; + logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); nloadbalancing = 0; } + ~LoadBalancer() { + logs.close(); + } + /** * This function reorder cells across MPI processes based on a * Space Filling Curve. This is mandatory for load balancing using SFC. @@ -621,6 +624,10 @@ namespace samurai std::string lbn = static_cast(this)->getName(); + logs << fmt::format("\n###################################################" ) << std::endl; + logs << fmt::format("> Load balancing ({}) mesh @ iteration {} ", lbn, nloadbalancing ) << std::endl; + logs << fmt::format("###################################################\n") << std::endl; + auto p = lbn.find( "SFC" ); if( nloadbalancing == 0 && p != std::string::npos ) { reordering( mesh , field, kw... ); @@ -993,7 +1000,6 @@ namespace samurai }); if (nbInter_ > 0){ - std::cerr << " not empty " << std::endl; minLevelAtInterface = std::min( projlevel, minLevelAtInterface ); } } @@ -1328,7 +1334,7 @@ namespace samurai // DEBUG std::ofstream logs; logs.open("log_" + std::to_string(world.rank()) + ".dat", std::ofstream::app); - logs << "# discover_neighbour" << std::endl; + logs << "\n# [LoadBalancer] discover_neighbour" << std::endl; // give access to geometricaly neighbour process rank and mesh std::vector& neighbourhood = mesh.mpi_neighbourhood(); @@ -1350,13 +1356,13 @@ namespace samurai if (!keepNeighbour[nbi]) { - logs << fmt::format("Loosing neighbour connection with {}", neighbourhood[nbi].rank) << std::endl; + logs << fmt::format("\t> Loosing neighbour connection with {}", neighbourhood[nbi].rank) << std::endl; } // check neighbour - neighbour connection for (size_t nbj = nbi + 1; nbj < neighbourhood.size(); ++nbj) { - logs << fmt::format("Checking neighbourhood connection {} <-> {}", neighbourhood[nbi].rank, neighbourhood[nbj].rank) + logs << fmt::format("\t> Checking neighbourhood connection {} <-> {}", neighbourhood[nbi].rank, neighbourhood[nbj].rank) << std::endl; bool connected = intersectionExists(neighbourhood[nbi].mesh, neighbourhood[nbj].mesh); @@ -1421,7 +1427,7 @@ namespace samurai } // debug - logs << "New neighbourhood : {"; + logs << "\t> New neighbourhood : {"; for (size_t nbi = 0; nbi < neighbourhood.size(); ++nbi) { logs << neighbourhood[nbi].rank << ", "; diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 38beeb0f9..94e28267a 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -51,7 +51,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( "keys", mesh ); @@ -60,7 +60,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer("rank", mesh); flags.fill( world.rank() ); - logs << fmt::format("\t\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; + logs << fmt::format("\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; SFC_key_t mink = std::numeric_limits::max(), maxk = std::numeric_limits::min(); @@ -89,12 +89,12 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer bounds = { mink, maxk }; - logs << "\t\t\t> Local key bounds [" << bounds[ 0 ] << ", " << bounds[ 1 ] << "]" << std::endl; + logs << "\t\t> Local key bounds [" << bounds[ 0 ] << ", " << bounds[ 1 ] << "]" << std::endl; std::vector boundaries; boost::mpi::all_gather( world, bounds.data(), bounds.size(), boundaries ); - logs << "\t\t\t> Global key boundaries ["; + logs << "\t\t> Global key boundaries ["; for(const auto & ik : boundaries ) logs << ik << ","; logs << "]" << std::endl; @@ -119,7 +119,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Global key evenly distrib boundaries ["; + logs << "\t\t> Global key evenly distrib boundaries ["; for(const auto & ik : boundaries_new ) logs << ik << ","; logs << "]" << std::endl; @@ -145,7 +145,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Comm required with processes : ["; + logs << "\t\t> Communication for exchange required with rank : ["; for( const auto & it : comm ) logs << it.first << ","; logs << "]" << std::endl; @@ -227,7 +227,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer sfc_map; @@ -237,7 +238,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer("rank", mesh); flags.fill( world.rank() ); - logs << fmt::format("\t\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; + logs << fmt::format("\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; SFC_key_t mink = std::numeric_limits::max(), maxk = std::numeric_limits::min(); size_t ncells = 0; @@ -277,7 +278,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer globIdx( world.size() + 1, 0 ), globIdxNew( world.size() + 1, 0 ); boost::mpi::all_gather( world, mesh.nb_cells( Mesh_t::mesh_id_t::cells ), nbCellsPerProc ); - logs << "\t\t\t> Number of cells : " << mesh.nb_cells( Mesh_t::mesh_id_t::cells ) << std::endl; + logs << "\t\t> Number of cells : " << mesh.nb_cells( Mesh_t::mesh_id_t::cells ) << std::endl; for(size_t i=0; i GlobalIdx : "; + logs << "\t\t> GlobalIdx : "; for(const auto & i : globIdx ) logs << i << ", "; logs << std::endl; - logs << "\t\t\t> GlobalIdx balanced : "; + logs << "\t\t> GlobalIdx balanced : "; for(const auto & i : globIdxNew ) logs << i << ", "; logs << std::endl; } - size_t start = 0; - - // for( size_t ip=0; ip globIdx[ world.rank() ] ) break; - // start ++; - // } - // start = std::min( start, static_cast( world.size() - 1 ) ); + int start = 0; while( globIdx[ world.rank() ] >= ( start + 1 ) * dc ){ start ++; } - logs << "Start @ rank " << start << std::endl; + logs << "\t\t> Start @ rank " << start << std::endl; size_t count = globIdx[ world.rank() ]; for( auto & it : sfc_map ) { if( count >= ( start + 1 ) * dc ){ start ++; - logs << "Incrementing Start @ rank " << start << ", count " << count << std::endl; + start = std::min( world.size() - 1 , start ); + logs << "\t\t> Incrementing Start @ rank " << start << ", count " << count << std::endl; } flags[ it.second ] = start; @@ -345,7 +340,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Comm required with processes : ["; + logs << "\t\t> Comm required with processes : ["; for( const auto & it : comm ) logs << it.first << ","; logs << "]" << std::endl; From cfd11cae13d99c843601f5b590b9bb7ddb9fac12 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 30 May 2024 13:27:57 +0200 Subject: [PATCH 100/170] add ordering function to others lb strategy --- include/samurai/load_balancing_diffusion.hpp | 8 +++++++- include/samurai/load_balancing_diffusion_cell.hpp | 9 +++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index c67465dca..2f2b1f51d 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -36,6 +36,11 @@ namespace Load_balancing{ inline std::string getName() const { return "diffusion"; } + template + Mesh_t reordering_impl( Mesh_t & mesh ) { + return mesh; + } + template Mesh_t load_balance_impl( Mesh_t & mesh ){ @@ -55,7 +60,8 @@ namespace Load_balancing{ logs << fmt::format("> New load-balancing using {} ", getName() ) << std::endl; // compute fluxes in terms of number of intervals to transfer/receive - std::vector fluxes = samurai::cmptFluxes( mesh ); + // by default, perform 5 iterations + std::vector fluxes = samurai::cmptFluxes( mesh, 5 ); std::vector new_fluxes( fluxes ); // get loads from everyone diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index f8fe88119..08f11498b 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -45,6 +45,11 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer + Mesh_t reordering_impl( Mesh_t & mesh ){ + return mesh; + } + template Mesh_t load_balance_impl( Mesh_t & mesh ){ @@ -70,8 +75,8 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( samurai::cmptLoad( mesh ) ); boost::mpi::all_gather( world, my_load, loads ); - // get the load to neighbours (geometrical neighbour) - std::vector fluxes = samurai::cmptFluxes( mesh ); + // get the load to neighbours (geometrical neighbour) with 5 iterations max + std::vector fluxes = samurai::cmptFluxes( mesh, 5 ); { logs << "load : " << my_load << std::endl; From 5dab244db0db65278544401d2aeebb71846e36a6 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 3 Jun 2024 10:56:19 +0200 Subject: [PATCH 101/170] fix MPI comm in SFC --- include/samurai/load_balancing_sfc.hpp | 92 ++++++++++++++++---------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 94e28267a..45068ad2a 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -101,7 +101,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer boundaries_new( world.size() + 1 ); + std::vector boundaries_new( static_cast( world.size() + 1 ) ); // find max value for boundaries SFC_key_t globalMax = boundaries[ 0 ]; @@ -113,9 +113,9 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( world.size() ); boundaries_new[ 0 ] = 0; - for(size_t ip=1; ip( world.size() ); ++ip ){ if( key >= boundaries_new[ ip ] && key < boundaries_new[ ip + 1 ] ){ flags[ cell ] = static_cast( ip ); @@ -155,7 +155,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer payload( world.size() ); + std::vector payload( static_cast( world.size() ) ); samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ @@ -164,9 +164,9 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } } }); @@ -182,7 +182,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( iproc ) ], false }; world.send( iproc, 17, to_send ); } @@ -218,7 +218,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Mesh_t load_balance_impl( Mesh_t & mesh ) { - using inter_t = samurai::Interval; using CellList_t = typename Mesh_t::cl_type; using CellArray_t = samurai::CellArray; @@ -241,7 +240,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; SFC_key_t mink = std::numeric_limits::max(), maxk = std::numeric_limits::min(); - size_t ncells = 0; samurai::for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto & cell ) { // this is where things can get nasty, we expect indices to be positive values !! @@ -267,28 +265,27 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer nbCellsPerProc; - std::vector globIdx( world.size() + 1, 0 ), globIdxNew( world.size() + 1, 0 ); + std::vector globIdx( static_cast( world.size() + 1 ) ); + std::vector globIdxNew( static_cast( world.size() + 1 ) ); boost::mpi::all_gather( world, mesh.nb_cells( Mesh_t::mesh_id_t::cells ), nbCellsPerProc ); logs << "\t\t> Number of cells : " << mesh.nb_cells( Mesh_t::mesh_id_t::cells ) << std::endl; - for(size_t i=0; i( world.size() ); ++i){ globIdx[ i + 1 ] = globIdx[ i ] + nbCellsPerProc[ i ]; ncells_tot += nbCellsPerProc[ i ]; } - dc = ncells_tot / world.size(); + dc = ncells_tot / static_cast( world.size() ); // load balanced globIdx globIdxNew[ 0 ] = 0; - for(size_t i=0; i( world.size() ); ++i){ globIdxNew[ i + 1 ] = globIdxNew[ i ] + dc; } @@ -306,13 +303,13 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer= ( start + 1 ) * dc ){ + while( globIdx[ static_cast( world.rank() ) ] >= static_cast( start + 1 ) * dc ){ start ++; } logs << "\t\t> Start @ rank " << start << std::endl; - size_t count = globIdx[ world.rank() ]; + size_t count = globIdx[ static_cast( world.rank() ) ]; for( auto & it : sfc_map ) { if( count >= ( start + 1 ) * dc ){ @@ -350,7 +347,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer payload( world.size() ); + std::vector payload( static_cast( world.size() ) ); samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ @@ -359,50 +356,75 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( flags[ cell ] ) < payload.size() ); + if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } } }); + std::vector req_send( world.size(), 0 ), req_recv( world.size(), 0 ); + for( int iproc=0; iproc send {} to # {}", reqExchg, iproc ) << std::endl; + req_send[ iproc ] = reqExchg; + + logs << fmt::format("\t\t\t\t> send {} to # {}", reqExchg, iproc ) << std::endl; world.send( iproc, 17, reqExchg ); - if( reqExchg == 1 ) { - CellArray_t to_send = { payload[ iproc ], false }; - world.send( iproc, 17, to_send ); - } + // if( reqExchg == 1 ) { + // CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; + // world.send( iproc, 17, to_send ); + // } } + logs << "\t> # Data sent to processes .... #" << std::endl; + for( int iproc=0; iproc recv {} to # {}", reqExchg, iproc ) << std::endl; + logs << fmt::format("\t\t\t\t> recv {} from # {}", reqExchg, iproc ) << std::endl; - if( reqExchg == 1 ) { + // if( reqExchg == 1 ) { + // CellArray_t to_rcv; + // world.recv( iproc, 17, to_rcv ); + + // samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ + // new_cl[ level ][ index ].add_interval( interval ); + // }); + // } + + } + + for(size_t iproc=0; iproc( iproc ) ], false }; + world.send( iproc, 17, to_send ); + } + + if( req_recv[ iproc ] == 1 ) { CellArray_t to_rcv; world.recv( iproc, 17, to_rcv ); - + samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ new_cl[ level ][ index ].add_interval( interval ); }); } - } - // /* ---------------------------------------------------------------------------------------------------------- */ // /* ------- Construct new mesh for current process ----------------------------------------------------------- */ // /* ---------------------------------------------------------------------------------------------------------- */ From 697101df05a876a4d9d4482c023fb445807fe5a8 Mon Sep 17 00:00:00 2001 From: Loic Gouarin Date: Fri, 31 May 2024 14:22:37 +0200 Subject: [PATCH 102/170] Add functions to compute the number of intervals in CellArray --- include/samurai/cell_array.hpp | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/include/samurai/cell_array.hpp b/include/samurai/cell_array.hpp index ce7d625c7..bab709d4a 100644 --- a/include/samurai/cell_array.hpp +++ b/include/samurai/cell_array.hpp @@ -120,6 +120,9 @@ namespace samurai std::size_t nb_cells() const; std::size_t nb_cells(std::size_t level) const; + std::size_t nb_intervals(std::size_t dim) const; + std::size_t nb_intervals(std::size_t dim, std::size_t level) const; + std::size_t max_level() const; std::size_t min_level() const; @@ -384,6 +387,38 @@ namespace samurai return m_cells[level].nb_cells(); } + /** + * Return the number of intervals for a given dimension over the levels. + * + * @param d The dimension where to compute the number of intervals + * @return The number of intervals for the given dimension + */ + template + inline std::size_t CellArray::nb_intervals(std::size_t d) const + { + assert(d < dim); + std::size_t size = 0; + for (std::size_t level = 0; level <= max_size; ++level) + { + size += m_cells[level].nb_intervals(d); + } + return size; + } + + /** + * Return the number of intervals for a given dimension and a given level. + * + * @param d The dimension where to compute the number of intervals + * @param level The level where to compute the number of intervals + * @return The number of intervals for the given dimension + */ + template + inline std::size_t CellArray::nb_intervals(std::size_t d, std::size_t level) const + { + assert(d < dim); + return m_cells[level].nb_intervals(d); + } + /** * Return the maximum level where the array entry is not empty. */ From 0f0016492d53acf9fe9754b7b23b8ecb27e0b56d Mon Sep 17 00:00:00 2001 From: Loic Gouarin Date: Fri, 31 May 2024 14:28:11 +0200 Subject: [PATCH 103/170] Add get_interval_location function in CellArray which return the index of the interval into the vector --- include/samurai/cell_array.hpp | 2 -- include/samurai/level_cell_array.hpp | 49 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/include/samurai/cell_array.hpp b/include/samurai/cell_array.hpp index bab709d4a..f44e5886c 100644 --- a/include/samurai/cell_array.hpp +++ b/include/samurai/cell_array.hpp @@ -96,10 +96,8 @@ namespace samurai template const interval_t& get_interval(std::size_t level, const interval_t& interval, T... index) const; - template const interval_t& get_interval(std::size_t level, const interval_t& interval, const xt::xexpression& index) const; - template const interval_t& get_interval(std::size_t level, const xt::xexpression& coord) const; diff --git a/include/samurai/level_cell_array.hpp b/include/samurai/level_cell_array.hpp index 9b62a9179..a69baa50b 100644 --- a/include/samurai/level_cell_array.hpp +++ b/include/samurai/level_cell_array.hpp @@ -119,6 +119,14 @@ namespace samurai /// Display to the given stream void to_stream(std::ostream& os) const; + // get_interval_location + template ...>, void>> + std::size_t get_interval_location(const interval_t& interval, T... index) const; + template + std::size_t get_interval_location(const interval_t& interval, const xt::xexpression& index) const; + template + std::size_t get_interval_location(const xt::xexpression& coord) const; + // get_interval template ...>, void>> const interval_t& get_interval(const interval_t& interval, T... index) const; @@ -151,6 +159,9 @@ namespace samurai //// Gives the total number of intervals auto nb_intervals() const; + //// Gives the number of intervals for a given dimension + std::size_t nb_intervals(std::size_t d) const; + //// Gives the number of cells std::size_t nb_cells() const; @@ -514,6 +525,37 @@ namespace samurai return const_reverse_iterator(cbegin()); } + /** + * Return the index in the x-interval array satisfying the input parameters + * + * @param interval The desired x-interval. + * @param index The desired indices for the other dimensions. + */ + template + template + inline std::size_t LevelCellArray::get_interval_location(const interval_t& interval, T... index) const + { + return static_cast(find(*this, {interval.start, index...})); + } + + template + template + inline std::size_t LevelCellArray::get_interval_location(const interval_t& interval, const xt::xexpression& index) const + { + xt::xtensor_fixed> point; + point[0] = interval.start; + xt::view(point, xt::range(1, _)) = index; + return static_cast(find(*this, point)); + } + + template + template + inline std::size_t LevelCellArray::get_interval_location(const xt::xexpression& coord) const + { + xt::xtensor_fixed> coord_array = coord; + return static_cast(find(*this, coord_array)); + } + /** * Return the x-interval satisfying the input parameters * @@ -640,6 +682,13 @@ namespace samurai return s; } + template + inline std::size_t LevelCellArray::nb_intervals(std::size_t d) const + { + assert(d < dim); + return m_cells[d].size(); + } + template inline std::size_t LevelCellArray::nb_cells() const { From be51e15552e0449044d101600a4716b96c03e0bc Mon Sep 17 00:00:00 2001 From: Loic Gouarin Date: Fri, 31 May 2024 14:29:35 +0200 Subject: [PATCH 104/170] Refactor Mesh_base class to include MPI neighborhood information as const --- include/samurai/mesh.hpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 3c4706cb2..36e8050c2 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -105,6 +105,7 @@ namespace samurai const std::array& periodicity() const; // std::vector& neighbouring_ranks(); + const std::vector& mpi_neighbourhood() const; std::vector& mpi_neighbourhood(); void swap(Mesh_base& mesh) noexcept; @@ -198,11 +199,11 @@ namespace samurai { ar& m_cells[id]; } - ar & m_domain; - ar & m_subdomain; - ar & m_union; - ar & m_min_level; - ar & m_max_level; + ar& m_domain; + ar& m_subdomain; + ar& m_union; + ar& m_min_level; + ar& m_max_level; } #endif }; @@ -490,9 +491,8 @@ namespace samurai template template - inline auto Mesh_base::get_interval(std::size_t level, - const interval_t& interval, - const xt::xexpression& index) const -> const interval_t& + inline auto Mesh_base::get_interval(std::size_t level, const interval_t& interval, const xt::xexpression& index) const + -> const interval_t& { return m_cells[mesh_id_t::reference].get_interval(level, interval, index); } @@ -558,6 +558,12 @@ namespace samurai return m_periodic; } + template + inline auto Mesh_base::mpi_neighbourhood() const -> const std::vector& + { + return m_mpi_neighbourhood; + } + template inline auto Mesh_base::mpi_neighbourhood() -> std::vector& { From cdce5392fe6f87c96199ac320bd08effaab753c5 Mon Sep 17 00:00:00 2001 From: Loic Gouarin Date: Fri, 31 May 2024 14:30:24 +0200 Subject: [PATCH 105/170] start adding metis and scotch for load balancing --- include/samurai/load_balancing_metis.hpp | 121 ++++++++++ include/samurai/load_balancing_scotch.hpp | 126 ++++++++++ include/samurai/load_balancing_utils.hpp | 267 ++++++++++++++++++++++ 3 files changed, 514 insertions(+) create mode 100644 include/samurai/load_balancing_metis.hpp create mode 100644 include/samurai/load_balancing_scotch.hpp create mode 100644 include/samurai/load_balancing_utils.hpp diff --git a/include/samurai/load_balancing_metis.hpp b/include/samurai/load_balancing_metis.hpp new file mode 100644 index 000000000..072a376de --- /dev/null +++ b/include/samurai/load_balancing_metis.hpp @@ -0,0 +1,121 @@ +// Copyright 2018-2024 the samurai's authors +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include + +#include "field.hpp" +#include "load_balancing_utils.hpp" + +namespace samurai::Load_balancing +{ + class Metis : public samurai::LoadBalancer + { + private: + + int _ndomains; + int _rank; + + template + void propagate(const Mesh_t&, const Stencil&, Field_t&, int, int&) const + { + } + + public: + + Metis() + { +#ifdef SAMURAI_WITH_MPI + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); +#else + _ndomains = 1; + _rank = 0; +#endif + } + + inline std::string getName() const + { + return "metis"; + } + + template + Mesh_t load_balance_impl(Mesh_t& mesh) + { + constexpr std::size_t dim = Mesh_t::dim; + using mesh_id_t = typename Mesh_t::mesh_id_t; + auto graph = build_graph(mesh); + + boost::mpi::communicator world; + + idx_t ndims = dim; + idx_t wgtflag = 1; + idx_t numflag = 0; + idx_t ncon = 1; + idx_t nparts = world.size(); + std::vector tpwgts(nparts, 1.0 / nparts); + real_t ubvec[] = {1.05}; + idx_t options[10] = {0}; + idx_t edgecut = 0; + std::vector part(mesh[mesh_id_t::cells].nb_cells(), 17); + MPI_Comm mpi_comm = MPI_COMM_WORLD; + + // ParMETIS_V3_PartKway(global_index_offset.data(), + // xadj.data(), + // adjncy.data(), + // NULL, + // NULL, // adjwgt.data(), + // &wgtflag, + // &numflag, + // &ncon, + // &nparts, + // tpwgts.data(), + // ubvec, + // options, + // &edgecut, + // part.data(), + // &mpi_comm); + + ParMETIS_V3_PartGeomKway(graph.global_index_offset.data(), + graph.xadj.data(), + graph.adjncy.data(), + NULL, + graph.adjwgt.data(), + &wgtflag, + &numflag, + &ndims, + graph.xyz.data(), + &ncon, + &nparts, + tpwgts.data(), + ubvec, + options, + &edgecut, + part.data(), + &mpi_comm); + + // ParMETIS_V3_AdaptiveRepart + + /* ---------------------------------------------------------------------------------------------------------- */ + /* ------- Construct new mesh for current process ----------------------------------------------------------- */ + /* ---------------------------------------------------------------------------------------------------------- */ + + auto partition_field = make_field("partition", mesh); + std::size_t ipart = 0; + for_each_cell(mesh, + [&](const auto& cell) + { + partition_field[cell] = part[ipart++]; + }); + // std::cout << "coucou" << std::endl; + save("metis_partition", mesh, partition_field); + // std::cout << "coucou 2" << std::endl; + + // Mesh_t new_mesh(new_cl, mesh); + + return mesh; + } + }; +} diff --git a/include/samurai/load_balancing_scotch.hpp b/include/samurai/load_balancing_scotch.hpp new file mode 100644 index 000000000..83f36c6cb --- /dev/null +++ b/include/samurai/load_balancing_scotch.hpp @@ -0,0 +1,126 @@ +// Copyright 2018-2024 the samurai's authors +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include "ptscotch.h" + +#include "field.hpp" +#include "load_balancing.hpp" +#include "load_balancing_utils.hpp" +#include "mesh_interval.hpp" +#include "stencil.hpp" + +using namespace xt::placeholders; + +namespace samurai::Load_balancing +{ + class Scotch : public samurai::LoadBalancer + { + private: + + int _ndomains; + int _rank; + + template + void propagate(const Mesh_t&, const Stencil&, Field_t&, int, int&) const + { + } + + public: + + Scotch() + { +#ifdef SAMURAI_WITH_MPI + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); +#else + _ndomains = 1; + _rank = 0; +#endif + } + + inline std::string getName() const + { + return "Scotch"; + } + + template + Mesh_t& reordering_impl(Mesh_t& mesh) + { + return mesh; + } + + template + Mesh_t load_balance_impl(Mesh_t& mesh) + { + constexpr std::size_t dim = Mesh_t::dim; + using mesh_id_t = typename Mesh_t::mesh_id_t; + auto graph = build_graph(mesh); + + boost::mpi::communicator world; + SCOTCH_Dgraph grafdat; + SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD); + + // std::cout << "xadj: " << xt::adapt(graph.xadj) << std::endl; + // std::cout << "adjnct: " << xt::adapt(graph.adjncy) << std::endl; + SCOTCH_dgraphBuild(&grafdat, // grafdat + 0, // baseval, c-style numbering + graph.xadj.size() - 1, // vertlocnbr, nCells + graph.xadj.size() - 1, // vertlocmax + const_cast(graph.xadj.data()), + nullptr, + + nullptr, // veloloctab, vtx weights + nullptr, // vlblloctab + + graph.adjncy.size(), // edgelocnbr, number of arcs + graph.adjncy.size(), // edgelocsiz + const_cast(graph.adjncy.data()), // edgeloctab + nullptr, // edgegsttab + nullptr // edlotab, edge weights + ); + SCOTCH_dgraphCheck(&grafdat); + SCOTCH_Strat strategy; + SCOTCH_stratInit(&strategy); + + SCOTCH_Arch archdat; + SCOTCH_archInit(&archdat); + // SCOTCH_archCmplt(&archdat, 1); + + std::vector part(mesh[mesh_id_t::cells].nb_cells(), 17); + + SCOTCH_Dmapping mappdat; + SCOTCH_dgraphMapInit(&grafdat, &mappdat, &archdat, part.data()); + + SCOTCH_dgraphPart(&grafdat, + 2, + &strategy, // const SCOTCH_Strat * + part.data() // parttab + ); + SCOTCH_archExit(&archdat); + SCOTCH_stratExit(&strategy); + SCOTCH_dgraphExit(&grafdat); + + /* ---------------------------------------------------------------------------------------------------------- */ + /* ------- Construct new mesh for current process ----------------------------------------------------------- */ + /* ---------------------------------------------------------------------------------------------------------- */ + + auto partition_field = make_field("partition", mesh); + std::size_t ipart = 0; + for_each_cell(mesh[mesh_id_t::cells], + [&](const auto& cell) + { + partition_field[cell] = part[ipart++]; + }); + // std::cout << "coucou" << std::endl; + save("metis_partition", mesh, partition_field); + // std::cout << "coucou 2" << std::endl; + + // Mesh_t new_mesh(new_cl, mesh); + + return mesh; + } + }; +} diff --git a/include/samurai/load_balancing_utils.hpp b/include/samurai/load_balancing_utils.hpp new file mode 100644 index 000000000..0af4acba3 --- /dev/null +++ b/include/samurai/load_balancing_utils.hpp @@ -0,0 +1,267 @@ +// Copyright 2018-2024 the samurai's authors +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include +#include +#include + +#include "ptscotch.h" + +#include "mesh_interval.hpp" +#include "stencil.hpp" + +using namespace xt::placeholders; + +namespace samurai +{ + namespace detail + { + template + inline void for_each_interface_impl(const Mesh& mesh1, const Mesh& mesh2, Func&& func) + { + constexpr std::size_t dim = Mesh::dim; + using interval_t = typename Mesh::interval_t; + using mesh_interval_t = MeshInterval; + const std::size_t max_level_jump = 1; + + auto directions = cartesian_directions(); + + for (std::size_t l_jump = 0; l_jump <= max_level_jump; ++l_jump) + { + if (mesh1.max_level() - mesh1.min_level() >= l_jump) + { + for (std::size_t level = mesh1.min_level(); level <= mesh1.max_level() - l_jump; ++level) + { + for (std::size_t id = 0; id < directions.shape(0); ++id) + { + auto d = xt::view(directions, id); + auto set = intersection(mesh1[level], translate(mesh2[level + l_jump], d)).on(level + l_jump); + set( + [&](const auto& interval, const auto& index) + { + mesh_interval_t mesh_interval_from{level + l_jump, interval - d(0), index - xt::view(d, xt::range(1, _))}; + mesh_interval_t mesh_interval_to{level, interval >> l_jump, index >> l_jump}; + mesh_interval_from.i.step = (1 << l_jump); + for (int ll = 0; ll < (1 << l_jump); ++ll) + { + if (mesh_interval_from.i.start == mesh_interval_from.i.end) + { + break; + } + func(mesh_interval_from, mesh_interval_to); + mesh_interval_from.i.start++; + } + }); + } + } + } + } + } + } + + template + inline void for_each_inner_interface(Mesh& mesh, Func&& func) + { + using mesh_id_t = typename Mesh::mesh_id_t; + + detail::for_each_interface_impl(mesh[mesh_id_t::cells], mesh[mesh_id_t::cells], std::forward(func)); + } + + template + inline void for_each_subdomain_interface(Mesh& mesh, Func&& func) + { + using mesh_id_t = typename Mesh::mesh_id_t; + + for (auto& neigh : mesh.mpi_neighbourhood()) + { + detail::for_each_interface_impl(mesh[mesh_id_t::cells], neigh.mesh[mesh_id_t::cells], std::forward(func)); + } + } + + template + inline auto get_local_interval_location(const Mesh& mesh, const MeshInterval& mesh_interval) + { + std::size_t level = mesh_interval.level; + auto& i = mesh_interval.i; + auto& index = mesh_interval.index; + std::size_t offset = 0; + for (std::size_t ll = 0; ll < level; ++ll) + { + offset += mesh[ll].nb_intervals(0); + } + return mesh[level].get_interval_location(i, index) + offset; + } + + template + inline auto get_local_start_interval(const Mesh& mesh, const MeshInterval& mesh_interval, const LocalIndex& local_index) + { + return local_index[get_local_interval_location(mesh, mesh_interval)] + mesh_interval.i.start; + } + + template + struct graph + { + std::vector global_index_offset; + std::vector xadj; + std::vector adjncy; + std::vector adjwgt; + std::vector xyz; + }; + + template + auto build_graph(const Mesh_t& all_meshes) + { + constexpr std::size_t dim = Mesh_t::dim; + using mesh_id_t = typename Mesh_t::mesh_id_t; + + // graph g; + graph g; + + boost::mpi::communicator world; + + auto& mesh = all_meshes[mesh_id_t::cells]; + + std::vector global_sizes(world.size()); + mpi::all_gather(world, mesh.nb_cells(), global_sizes); + + g.global_index_offset.resize(world.size() + 1); + for (std::size_t i = 1; i < g.global_index_offset.size(); ++i) + { + g.global_index_offset[i] = g.global_index_offset[i - 1] + global_sizes[i - 1]; + } + + // Construct the local index numbering of the cells + std::vector local_index(mesh.nb_intervals(0)); + std::size_t counter = 0; + std::size_t current_size = 0; + for_each_interval(mesh, + [&](std::size_t level, const auto& i, auto& index) + { + local_index[counter++] = current_size - i.start; + current_size += i.size(); + }); + + // Construct the list adjacent cells for each cell + std::map> adj_cells; + std::map> adj_weights; + std::size_t data_size = 0; + + // interior interface + for_each_inner_interface( + all_meshes, + [&](const auto& mesh_interval_from, const auto& mesh_interval_to) + { + auto start_from = get_local_start_interval(mesh, mesh_interval_from, local_index) + g.global_index_offset[world.rank()]; + auto start_to = get_local_start_interval(mesh, mesh_interval_to, local_index) + g.global_index_offset[world.rank()]; + + // std::cout << "From: " << mesh_interval_from.level << " " << mesh_interval_from.i << " " << mesh_interval_from.index + // << std::endl; + // std::cout << "To: " << mesh_interval_to.level << " " << mesh_interval_to.i << " " << mesh_interval_to.index << std::endl; + auto step = mesh_interval_from.i.step; + for (int i = 0; i < mesh_interval_to.i.size(); ++i) + { + // std::cout << i << " " << step << " " << start_from + i * step << " " << start_to + i << std::endl; + adj_cells[start_from + i * step].push_back(start_to + i); + adj_weights[start_from + i * step].push_back(0.5 * (mesh_interval_from.level + mesh_interval_to.level)); + data_size++; + if (mesh_interval_from.level != mesh_interval_to.level) + { + adj_cells[start_to + i].push_back(start_from + i * step); + adj_weights[start_to + i].push_back(0.5 * (mesh_interval_from.level + mesh_interval_to.level)); + data_size++; + } + } + // std::cout << std::endl; + }); + + std::vector req; + std::vector> to_send(all_meshes.mpi_neighbourhood().size()); + std::set recv_from; + + std::size_t i_neigh = 0; + for (auto& neigh : all_meshes.mpi_neighbourhood()) + { + detail::for_each_interface_impl(mesh, + neigh.mesh[mesh_id_t::cells], + [&](const auto&, const auto& mesh_interval) + { + recv_from.insert(neigh.rank); + auto start = get_local_start_interval(mesh, mesh_interval, local_index) + + g.global_index_offset[world.rank()]; + to_send[i_neigh].push_back(static_cast(start)); + }); + if (recv_from.find(neigh.rank) != recv_from.end()) + { + req.push_back(world.isend(neigh.rank, neigh.rank, to_send[i_neigh++])); + } + } + + for (auto& neigh : all_meshes.mpi_neighbourhood()) + { + if (recv_from.find(neigh.rank) != recv_from.end()) + { + std::vector to_recv; + world.recv(neigh.rank, world.rank(), to_recv); + + std::size_t to_recv_counter = 0; + detail::for_each_interface_impl( + neigh.mesh[mesh_id_t::cells], + mesh, + [&](const auto& mesh_interval, const auto& mesh_interval_to) + { + auto start_1 = get_local_start_interval(mesh, mesh_interval, local_index) + g.global_index_offset[world.rank()]; + auto start_2 = to_recv[to_recv_counter++]; + + for (int i = 0, i1 = start_1, i2 = start_2; i < mesh_interval.i.size(); ++i, ++i1, ++i2) + { + adj_cells[i1].push_back(i2); + adj_weights[start_1 + i].push_back(0.5 * (mesh_interval.level + mesh_interval_to.level)); + data_size += 1; + } + }); + } + } + + mpi::wait_all(req.begin(), req.end()); + + // Construct the adjacency list in CSR format + g.xadj.resize(mesh.nb_cells() + 1); + + g.xadj[0] = 0; + std::size_t i_xadj = 0; + for (auto& v : adj_cells) + { + g.xadj[i_xadj + 1] = g.xadj[i_xadj] + v.second.size(); + ++i_xadj; + } + + g.adjncy.resize(g.xadj.back()); + g.adjwgt.resize(g.xadj.back()); + i_xadj = 0; + for (auto& v : adj_cells) + { + std::copy(v.second.begin(), v.second.end(), g.adjncy.begin() + g.xadj[i_xadj]); + ++i_xadj; + } + + i_xadj = 0; + for (auto& v : adj_weights) + { + std::copy(v.second.begin(), v.second.end(), g.adjwgt.begin() + g.xadj[i_xadj]); + ++i_xadj; + } + + std::cout << data_size << " " << g.xadj.back() << std::endl; + // assert(data_size == g.xadj.back()); + g.xyz.reserve(mesh.nb_cells() * dim); + for_each_cell(mesh, + [&](const auto& cell) + { + auto center = cell.center(); + std::copy(center.begin(), center.end(), std::back_inserter(g.xyz)); + }); + return g; + } +} From 15b056e71afea24b07a8bca0ce97a765b6ebcd3d Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 3 Jun 2024 10:58:56 +0200 Subject: [PATCH 106/170] fix warnings --- include/samurai/load_balancing.hpp | 19 ++++++++++--------- include/samurai/load_balancing_void.hpp | 5 ++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index f96cfc950..5baadae9c 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -113,7 +113,10 @@ namespace samurai None }; - static const double load_balancing_threshold = 0.03141592; // 2.5 % + static const double load_balancing_threshold = 0.03141592; // 2.5 % + // static const std::vector load_balancing_cell_weight = { 1., 0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, + // 0.0078125, 0.00390625, 0.001953125, 0.0009765625, + // 0.00048828125, 0.000244140625, 0.0001220703125 }; /** * Compute distance base on different norm. @@ -172,7 +175,7 @@ namespace samurai samurai::for_each_interval(current_mesh, [&]([[maybe_unused]] std::size_t level, const auto& interval, [[maybe_unused]] const auto& index) { - current_process_load += interval.size(); + current_process_load += interval.size(); // * load_balancing_cell_weight[ level ]; }); } else @@ -364,7 +367,6 @@ namespace samurai for (std::size_t n_i = 0; n_i < n_neighbours; ++n_i) { std::size_t neighbour_rank = static_cast( neighbourhood[ n_i ].rank ); - int neighbour_load = loads[neighbour_rank]; int abs_diff = std::abs( fluxes[ n_i ] ); int threshold_neigh = static_cast( load_balancing_threshold * loads[ neighbour_rank ] ); int threshold_curr = static_cast( load_balancing_threshold * my_load ); @@ -461,7 +463,7 @@ namespace samurai } std::vector req; - std::vector> to_send(world.size()); + std::vector> to_send( static_cast( world.size() ) ); std::size_t i_neigh = 0; @@ -474,7 +476,7 @@ namespace samurai // for (auto& neighbour : new_mesh.mpi_neighbourhood()) for( size_t ni=0; ni( ni ) == world.rank() ) continue; // auto & neighbour_new_mesh = neighbour.mesh; auto & neighbour_new_mesh = all_new_meshes[ ni ]; @@ -495,7 +497,7 @@ namespace samurai if (to_send[ni].size() != 0) { // neighbour_rank = neighbour.rank; - auto neighbour_rank = ni; + auto neighbour_rank = static_cast( ni ); req.push_back( world.isend( neighbour_rank, neighbour_rank, to_send[ ni ] ) ); // i_neigh ++; @@ -508,7 +510,7 @@ namespace samurai // build payload of field that I need to receive from neighbour, so compare NEW mesh with OLD neighbour mesh for (size_t ni=0; ni( ni ) == world.rank() ) continue; bool isintersect = false; for (std::size_t level = min_level; level <= max_level; ++level) @@ -516,7 +518,6 @@ namespace samurai if (!new_mesh[mesh_id_t::cells][level].empty() && !all_old_meshes[ ni ][mesh_id_t::cells][level].empty()) { std::vector to_recv; - std::ptrdiff_t count = 0; auto in_interface = intersection( all_old_meshes[ ni ][mesh_id_t::cells][level], new_mesh[mesh_id_t::cells][level]); @@ -537,7 +538,7 @@ namespace samurai { std::ptrdiff_t count = 0; std::vector to_recv; - world.recv( ni, world.rank(), to_recv); + world.recv( static_cast( ni ), world.rank(), to_recv); for (std::size_t level = min_level; level <= max_level; ++level) { diff --git a/include/samurai/load_balancing_void.hpp b/include/samurai/load_balancing_void.hpp index 0def67d05..7d2cf4c57 100644 --- a/include/samurai/load_balancing_void.hpp +++ b/include/samurai/load_balancing_void.hpp @@ -31,6 +31,9 @@ class Void_LoadBalancer: public samurai::LoadBalancer> { inline std::string getName() const { return "Void_LB"; } template - void load_balance_impl( [[maybe_unused]] Mesh_t & mesh ){ } + Mesh_t reordering_impl( Mesh_t & mesh ) { return mesh; } + + template + Mesh_t load_balance_impl( Mesh_t & mesh ){ return mesh; } }; \ No newline at end of file From 38d5d680919e47775c8d4f65fd11921742b2e9de Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 3 Jun 2024 10:59:22 +0200 Subject: [PATCH 107/170] fix warnings --- include/samurai/load_balancing_diffusion.hpp | 2 +- include/samurai/load_balancing_diffusion_cell.hpp | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 2f2b1f51d..0747cf5f4 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -173,7 +173,7 @@ namespace Load_balancing{ bool empty = false; { size_t iii = 0; - samurai::for_each_interval( interface, [&](const auto & level, const auto & i, const auto ii ){ + samurai::for_each_interval( interface, [&]( [[maybe_unused]] size_t level, [[maybe_unused]] const auto & i, [[maybe_unused]] const auto & ii ){ iii ++; }); if( iii == 0 ) empty = true; diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 08f11498b..4b9684114 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -97,9 +97,10 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( mesh ); std::vector ncells_interface ( interface.size(), 0 ); - for(int ni=0; ni( world.rank() ) ]; double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; - std::vector mload( world.size(), 0 ); + std::vector mload( static_cast( world.size() ), 0 ); // select the neighbour for( std::size_t ni=0; ni Date: Mon, 3 Jun 2024 10:59:28 +0200 Subject: [PATCH 108/170] fix missings refs --- include/samurai/hdf5.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/samurai/hdf5.hpp b/include/samurai/hdf5.hpp index 94bbfc8ad..9d4a3b19b 100644 --- a/include/samurai/hdf5.hpp +++ b/include/samurai/hdf5.hpp @@ -93,7 +93,7 @@ namespace samurai { std::size_t index = 0; for_each_cell(submesh, - [&](auto cell) + [&](const auto & cell) { if constexpr (Field::size == 1) { @@ -136,7 +136,7 @@ namespace samurai std::size_t id = 0; std::size_t index = 0; for_each_cell(mesh, - [&](auto cell) + [&](const auto & cell) { std::array a; auto start_corner = cell.corner(); From ac604c1b4b56c7c51bdf45822e919313ff1dd775 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 3 Jun 2024 10:59:47 +0200 Subject: [PATCH 109/170] updarte demo adv --- demos/FiniteVolume/advection_2d.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 4610e7b05..47572ecd4 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include @@ -300,8 +301,8 @@ int main(int argc, char* argv[]) const int iof = 1; - SFC_LoadBalancer_interval balancer; - + SFC_LoadBalancer_interval balancer; + // Void_LoadBalancer balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; // Load_balancing::Diffusion balancer; From 85d842a610e3579dfb61eba9cd447db79ef41091 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 3 Jun 2024 16:33:50 +0200 Subject: [PATCH 110/170] fix warnings --- include/samurai/load_balancing_sfc.hpp | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 45068ad2a..f2e7c951b 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -364,7 +364,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer req_send( world.size(), 0 ), req_recv( world.size(), 0 ); + std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); for( int iproc=0; iproc( iproc ) ] = reqExchg; - logs << fmt::format("\t\t\t\t> send {} to # {}", reqExchg, iproc ) << std::endl; world.send( iproc, 17, reqExchg ); // if( reqExchg == 1 ) { @@ -392,9 +391,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer recv {} from # {}", reqExchg, iproc ) << std::endl; + world.recv( iproc, 17, req_recv[ static_cast( iproc ) ] ); // if( reqExchg == 1 ) { // CellArray_t to_rcv; @@ -407,15 +404,15 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( iproc ) ] == 1 ){ CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; world.send( iproc, 17, to_send ); } - if( req_recv[ iproc ] == 1 ) { + if( req_recv[ static_cast( iproc ) ] == 1 ) { CellArray_t to_rcv; world.recv( iproc, 17, to_rcv ); @@ -539,7 +536,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer 0) @@ -548,7 +544,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancersecond.given = true; // flag "interval" has been sent my_load_i -= 1; transfer_load_prev += 1; - niter_send++; } else { @@ -592,7 +587,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer 0) @@ -601,7 +595,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancersecond.given = true; my_load_i -= 1; transfer_load_next += 1; - niter_send++; } else { From 0abf0e77f715c496b978210534dd3d68263e16c3 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 3 Jun 2024 16:57:03 +0200 Subject: [PATCH 111/170] fix warnings --- include/samurai/load_balancing.hpp | 4 --- include/samurai/load_balancing_diffusion.hpp | 8 +----- include/samurai/load_balancing_sfc.hpp | 26 ++------------------ 3 files changed, 3 insertions(+), 35 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 5baadae9c..8d5e1ef0c 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -270,7 +270,6 @@ namespace samurai for (std::size_t n_i = 0; n_i < n_neighbours; ++n_i) { std::size_t neighbour_rank = static_cast( neighbourhood[ n_i ] ); - int neighbour_load = loads[neighbour_rank]; int abs_diff = std::abs( fluxes[ n_i ] ); int threshold_neigh = static_cast( load_balancing_threshold * loads[ neighbour_rank ] ); int threshold_curr = static_cast( load_balancing_threshold * my_load ); @@ -465,8 +464,6 @@ namespace samurai std::vector req; std::vector> to_send( static_cast( world.size() ) ); - std::size_t i_neigh = 0; - // FIXME: this is overkill and will not scale std::vector all_new_meshes, all_old_meshes; boost::mpi::all_gather( world, new_mesh, all_new_meshes ); @@ -499,7 +496,6 @@ namespace samurai // neighbour_rank = neighbour.rank; auto neighbour_rank = static_cast( ni ); req.push_back( world.isend( neighbour_rank, neighbour_rank, to_send[ ni ] ) ); - // i_neigh ++; logs << fmt::format("\t> [LoadBalancer]::update_field send data to rank # {}", neighbour_rank ) << std::endl; } diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 0747cf5f4..9c048d3f9 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -15,11 +15,6 @@ namespace Load_balancing{ int _ndomains; int _rank; - template - void propagate( const Mesh_t & mesh, const Stencil & dir, Field_t & field, int value, int &given ) const { - - } - public: Diffusion() { @@ -187,10 +182,9 @@ namespace Load_balancing{ { size_t nCellsAtInterfaceGiven = 0, nCellsAtInterface = 0; for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { - size_t nIntervalAtInterface = 0; + auto intersect = samurai::intersection( interface[ interface.min_level() ], mesh[ mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! intersect( [&]( [[maybe_unused]] const auto & interval, [[maybe_unused]] const auto & index ){ - nIntervalAtInterface += 1; for(size_t ii=0; ii::max(), maxk = std::numeric_limits::min(); - size_t ncells = 0; samurai::for_each_cell(mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ) { // this is where things can get nasty, we expect indices to be positive values !! @@ -83,8 +82,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( iproc ) ], false }; - // world.send( iproc, 17, to_send ); - // } - } logs << "\t> # Data sent to processes .... #" << std::endl; for( int iproc=0; iproc( iproc ) ] ); - - // if( reqExchg == 1 ) { - // CellArray_t to_rcv; - // world.recv( iproc, 17, to_rcv ); - - // samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ - // new_cl[ level ][ index ].add_interval( interval ); - // }); - // } - + world.recv( iproc, 17, req_recv[ static_cast( iproc ) ] ); } for(int iproc=0; iproc Computing SFC ({}) 1D indices (interval) ... ", _sfc.getName() ) << std::endl; - size_t ninterval = 0; + [[maybe_unused]] size_t ninterval = 0; samurai::for_each_interval(mesh, [&]( std::size_t level, const auto & inter, const auto & index ) { // get Logical coordinate or first cell xt::xtensor_fixed> icell; From 8747a0f586e6e466980ad8c47b5db67a29cde3d4 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 3 Jun 2024 16:57:15 +0200 Subject: [PATCH 112/170] fix init advection for 3D tests --- demos/FiniteVolume/advection_2d.cpp | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 47572ecd4..e214301f9 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -53,7 +53,7 @@ auto init(Mesh& mesh, const double radius, const double x_center, const double y } template -auto init(Mesh& mesh, const std::vector & radius, const std::vector & xcenters, const std::vector & ycenters ) +auto init(Mesh& mesh, const std::vector & radius, const std::vector & rcenters ) { auto u = samurai::make_field("u", mesh); @@ -66,7 +66,12 @@ auto init(Mesh& mesh, const std::vector & radius, const std::vectorcapture_default_str()->group("Simulation parameters"); app.add_option("--max-corner", max_corner, "The max corner of the box")->capture_default_str()->group("Simulation parameters"); @@ -272,19 +277,17 @@ int main(int argc, char* argv[]) const double dt_save = Tf / static_cast(nfiles); double t = 0.; - std::vector xcenters, ycenters, radii; - xcenters.emplace_back( x_center ); - ycenters.emplace_back( y_center ); + std::vector rcenters, radii; + rcenters.emplace_back( x_center ); + rcenters.emplace_back( y_center ); radii.emplace_back( radius ); - // xcenters = { 0.3, 1., 2.3 }; - // ycenters = { 0.3, 1.4, 3. }; - // radii = { 0.2, 0.23, 0.3 }; - xcenters = { 1., 3., 1. }; - ycenters = { 1., 1., 3. }; + rcenters = { 1., 1., + 1., 3., + 1., 3. }; radii = { 0.2, 0.2, 0.2 }; - auto u = init(mesh, radii, xcenters, ycenters); + auto u = init(mesh, radii, rcenters); samurai::make_bc>(u, 0.); auto unp1 = samurai::make_field("unp1", mesh); @@ -345,7 +348,7 @@ int main(int argc, char* argv[]) if (correction) { - flux_correction(dt, a, u, unp1); + // flux_correction(dt, a, u, unp1); } std::swap(u.array(), unp1.array()); From dc1414b9852c322a6cedfe5e4b740b7c1512f457 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 5 Jun 2024 11:39:46 +0200 Subject: [PATCH 113/170] add update mesh & change return object for LB to be a field with rank --- include/samurai/load_balancing.hpp | 85 ++++++++++++- .../samurai/load_balancing_diffusion_cell.hpp | 116 ++---------------- include/samurai/load_balancing_sfc.hpp | 92 +++++++------- 3 files changed, 138 insertions(+), 155 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 8d5e1ef0c..da8676433 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -615,6 +615,85 @@ namespace samurai } + // const field_t & not possible with flags[ cell ] why ? + template + Mesh_t update_mesh( Mesh_t & mesh, Field_t & flags ) { + + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; + + boost::mpi::communicator world; + + CellList_t new_cl; + std::vector payload( static_cast( world.size() ) ); + + std::map comm; + + // build cell list for the current process && cells lists of cells for other processes + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ + + if( flags[ cell ] == world.rank() ){ + if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + }else{ + assert( static_cast( flags[ cell ] ) < payload.size() ); + + if( comm.find( flags[ cell ] ) == comm.end() ) { + comm[ flags[ cell ] ] = true; + } + + if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + } + + }); + + std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); + + // Required to know communication pattern + for( int iproc=0; iproc( iproc ) ] = reqExchg; + + world.send( iproc, 17, reqExchg ); + + } + + for( int iproc=0; iproc( iproc ) ] ); + } + + // actual data echange between processes that need to exchange data + for(int iproc=0; iproc( iproc ) ] == 1 ){ + CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; + world.send( iproc, 17, to_send ); + } + + if( req_recv[ static_cast( iproc ) ] == 1 ) { + CellArray_t to_rcv; + world.recv( iproc, 17, to_rcv ); + + samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ + new_cl[ level ][ index ].add_interval( interval ); + }); + } + } + + Mesh_t new_mesh( new_cl, mesh ); + + return new_mesh; + } + template void load_balance(Mesh_t & mesh, Field_t& field, Fields&... kw) { @@ -631,7 +710,11 @@ namespace samurai } // specific load balancing strategy - auto new_mesh = static_cast(this)->load_balance_impl( field.mesh() ); + // auto new_mesh = static_cast(this)->load_balance_impl( field.mesh() ); + + auto flags = static_cast(this)->load_balance_impl( field.mesh() ); + + auto new_mesh = update_mesh( mesh, flags ); // update each physical field on the new load balanced mesh SAMURAI_TRACE("[LoadBalancer::load_balance]::Updating fields ... "); diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 4b9684114..fee288e70 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -20,7 +20,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer - Mesh_t load_balance_impl( Mesh_t & mesh ){ + auto load_balance_impl( Mesh_t & mesh ){ using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; using CellList_t = typename Mesh_t::cl_type; @@ -119,6 +119,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( interface[ nbi ] ); barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); + // surface or volume depending on dim sv[ nbi ] = getSurfaceOrVolume( neighbourhood[ nbi ].mesh ); // debug @@ -135,7 +136,8 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer repartition; + auto flags = samurai::make_field("rank", mesh); + flags.fill( world.rank() ); constexpr auto fdist = samurai::Distance_t::GRAVITY; @@ -181,116 +183,12 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer= 0 ){ - - if( repartition.find( winner_dist ) != repartition.find( winner_dist ) ){ - std::cerr << "\t> WARNING: Key conflict for std::map !" << std::endl; - } - - repartition.insert ( std::pair( winner_dist, Data { cell, static_cast( winner_id ) } ) ); - + flags[ cell ] = neighbourhood[ winner_id ].rank; } }); - - std::vector cl_to_send( n_neighbours ); - std::vector ca_to_send( n_neighbours ); - - // distribute intervals based on ordered distance to neighbours - // rank is not the rank but the offset in the neighbourhood - std::vector given_ ( n_neighbours, 0 ); - // for( auto & it : repartition ){ - for( auto it = repartition.begin(); it != repartition.end(); it++ ){ - - std::size_t rank = static_cast( it->second.rank ); - - // shouldn't we give it to the second closest neighbour ?! - if( given_[ rank ] + 1 <= ( - fluxes[ rank ] ) ){ - - if constexpr ( dim == 3 ) { - auto i = it->second.cell.indices[ 0 ]; - auto j = it->second.cell.indices[ 1 ]; - auto k = it->second.cell.indices[ 2 ]; - cl_to_send[ rank ][ it->second.cell.level ][ { j, k } ].add_point( i ); - }else{ - auto i = it->second.cell.indices[ 0 ]; - auto j = it->second.cell.indices[ 1 ]; - cl_to_send[ rank ][ it->second.cell.level ][ { j } ].add_point( i ); - } - - given_[ rank ] += 1; - - } - - } - - for(size_t nbi=0; nbi Number of cells to send to process # " << neighbourhood[ nbi ].rank - << " : " << ca_to_send[ nbi ].nb_cells() << std::endl; - } - - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Data transfer between processes ------------------------------------------------------------------ */ - /* ---------------------------------------------------------------------------------------------------------- */ - - CellList_t new_cl, need_remove; - for(size_t ni=0; ni 0 ) { // receive data - samurai::CellArray to_rcv; - world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); - - logs << "Receiving data from # " << neighbourhood[ ni ].rank - << ", nbCells : " << to_rcv.nb_cells() << std::endl; - - // old strategy: modifying the current mesh - not working, breaks some internals - // mesh.merge( to_rcv ); - - // new strategy: build a whole new mesh from a cl_t - samurai::for_each_interval(to_rcv, - [&](std::size_t level, const auto& interval, const auto& index) - { - new_cl[ level ][ index ].add_interval( interval ); - }); - - }else{ // send data to - world.send( neighbourhood[ ni ].rank, 42, ca_to_send[ ni ] ); - - logs << "Sending data to # " << neighbourhood[ ni ].rank - << ", nbCells : " << ca_to_send[ ni ].nb_cells() << std::endl; - - // old strategy: modifying the current mesh - not working, breaks some internals - // mesh.remove( ca_to_send[ ni ] ); - - samurai::for_each_interval(ca_to_send[ ni ], - [&](std::size_t level, const auto& interval, const auto& index) - { - need_remove[ level ][ index ].add_interval( interval ); - }); - - } - } - - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Construct new mesh for current process ----------------------------------------------------------- */ - /* ---------------------------------------------------------------------------------------------------------- */ - - // add to new_cl interval that were not sent - CellArray_t need_remove_ca = { need_remove }; // to optimize - for( size_t level=mesh.min_level(); level<=mesh.max_level(); ++level ){ - auto diff = samurai::difference( mesh[ mesh_id_t::cells ][ level ], need_remove_ca[ level ] ); - - diff([&]( auto & interval, auto & index ){ - new_cl[ level ][ index ].add_interval( interval ); - }); - } - - Mesh_t new_mesh( new_cl, mesh ); - - return new_mesh; + return flags; } }; \ No newline at end of file diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index e8df6f97b..b0b0b1594 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -89,7 +89,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Local key bounds [" << bounds[ 0 ] << ", " << bounds[ 1 ] << "]" << std::endl; std::vector boundaries; - boost::mpi::all_gather( world, bounds.data(), bounds.size(), boundaries ); + boost::mpi::all_gather( world, bounds.data(), static_cast( bounds.size() ), boundaries ); logs << "\t\t> Global key boundaries ["; for(const auto & ik : boundaries ) @@ -213,7 +213,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer - Mesh_t load_balance_impl( Mesh_t & mesh ) + auto load_balance_impl( Mesh_t & mesh ) { using CellList_t = typename Mesh_t::cl_type; using CellArray_t = samurai::CellArray; @@ -343,70 +343,72 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer payload( static_cast( world.size() ) ); + // CellList_t new_cl; + // std::vector payload( static_cast( world.size() ) ); - samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ + // samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ - if( flags[ cell ] == world.rank() ){ - if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - }else{ - assert( static_cast( flags[ cell ] ) < payload.size() ); - if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - } + // if( flags[ cell ] == world.rank() ){ + // if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + // }else{ + // assert( static_cast( flags[ cell ] ) < payload.size() ); + // if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + // } - }); + // }); - std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); + // std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); - for( int iproc=0; iproc( iproc ) ] = reqExchg; + // req_send[ static_cast( iproc ) ] = reqExchg; - world.send( iproc, 17, reqExchg ); + // world.send( iproc, 17, reqExchg ); - } + // } - logs << "\t> # Data sent to processes .... #" << std::endl; + // logs << "\t> # Data sent to processes .... #" << std::endl; - for( int iproc=0; iproc( iproc ) ] ); - } + // for( int iproc=0; iproc( iproc ) ] ); + // } - for(int iproc=0; iproc( iproc ) ] == 1 ){ - CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; - world.send( iproc, 17, to_send ); - } + // if( req_send[ static_cast( iproc ) ] == 1 ){ + // CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; + // world.send( iproc, 17, to_send ); + // } - if( req_recv[ static_cast( iproc ) ] == 1 ) { - CellArray_t to_rcv; - world.recv( iproc, 17, to_rcv ); + // if( req_recv[ static_cast( iproc ) ] == 1 ) { + // CellArray_t to_rcv; + // world.recv( iproc, 17, to_rcv ); - samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ - new_cl[ level ][ index ].add_interval( interval ); - }); - } - } + // samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ + // new_cl[ level ][ index ].add_interval( interval ); + // }); + // } + // } // /* ---------------------------------------------------------------------------------------------------------- */ // /* ------- Construct new mesh for current process ----------------------------------------------------------- */ // /* ---------------------------------------------------------------------------------------------------------- */ - Mesh_t new_mesh( new_cl, mesh ); + // Mesh_t new_mesh( new_cl, mesh ); - return new_mesh; + // return new_mesh; + + return flags; } template From 30909e95cce51bfc923139696cec3208eabad8a1 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 5 Jun 2024 12:48:09 +0200 Subject: [PATCH 114/170] clean code and fix wrning --- include/samurai/load_balancing.hpp | 10 +- .../samurai/load_balancing_diffusion_cell.hpp | 4 +- include/samurai/load_balancing_sfc.hpp | 307 ------------------ 3 files changed, 9 insertions(+), 312 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index da8676433..b7343d68a 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -615,9 +615,8 @@ namespace samurai } - // const field_t & not possible with flags[ cell ] why ? template - Mesh_t update_mesh( Mesh_t & mesh, Field_t & flags ) { + Mesh_t update_mesh( Mesh_t & mesh, const Field_t & flags ) { using CellList_t = typename Mesh_t::cl_type; using CellArray_t = typename Mesh_t::ca_type; @@ -650,6 +649,11 @@ namespace samurai }); + logs << "\t\t>[Load_balancer::update_mesh] Comm required with processes : ["; + for( const auto & it : comm ) + logs << it.first << ","; + logs << "]" << std::endl; + std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); // Required to know communication pattern @@ -711,7 +715,7 @@ namespace samurai // specific load balancing strategy // auto new_mesh = static_cast(this)->load_balance_impl( field.mesh() ); - + auto flags = static_cast(this)->load_balance_impl( field.mesh() ); auto new_mesh = update_mesh( mesh, flags ); diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index fee288e70..0eacacada 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -55,7 +55,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer; + using CellArray_t = typename Mesh_t::ca_type; using Cell_t = typename Mesh_t::cell_t; using mesh_id_t = typename Mesh_t::mesh_id_t; @@ -183,7 +183,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer= 0 ){ - flags[ cell ] = neighbourhood[ winner_id ].rank; + flags[ cell ] = neighbourhood[ static_cast( winner_id ) ].rank; } }); diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index b0b0b1594..6a530b8dd 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -215,8 +215,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer auto load_balance_impl( Mesh_t & mesh ) { - using CellList_t = typename Mesh_t::cl_type; - using CellArray_t = samurai::CellArray; boost::mpi::communicator world; @@ -320,312 +318,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer comm; - samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&](const auto & cell ) { - - if( comm.find( flags[ cell ] ) == comm.end() ) { - comm[ flags[ cell ] ] = true; - } - - }); - - logs << "\t\t> Comm required with processes : ["; - for( const auto & it : comm ) - logs << it.first << ","; - logs << "]" << std::endl; - - // /* ---------------------------------------------------------------------------------------------------------- */ - // /* ------- Data transfer between processes ------------------------------------------------------------------ */ - // /* ---------------------------------------------------------------------------------------------------------- */ - - // CellList_t new_cl; - // std::vector payload( static_cast( world.size() ) ); - - // samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ - - // if( flags[ cell ] == world.rank() ){ - // if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - // }else{ - // assert( static_cast( flags[ cell ] ) < payload.size() ); - // if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - // } - - // }); - - // std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); - - // for( int iproc=0; iproc( iproc ) ] = reqExchg; - - // world.send( iproc, 17, reqExchg ); - - // } - - // logs << "\t> # Data sent to processes .... #" << std::endl; - - // for( int iproc=0; iproc( iproc ) ] ); - // } - - // for(int iproc=0; iproc( iproc ) ] == 1 ){ - // CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; - // world.send( iproc, 17, to_send ); - // } - - // if( req_recv[ static_cast( iproc ) ] == 1 ) { - // CellArray_t to_rcv; - // world.recv( iproc, 17, to_rcv ); - - // samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ - // new_cl[ level ][ index ].add_interval( interval ); - // }); - // } - // } - - // /* ---------------------------------------------------------------------------------------------------------- */ - // /* ------- Construct new mesh for current process ----------------------------------------------------------- */ - // /* ---------------------------------------------------------------------------------------------------------- */ - - // Mesh_t new_mesh( new_cl, mesh ); - - // return new_mesh; - return flags; } - template - Mesh_t load_balance_impl_old( Mesh_t& mesh ) - { - using inter_t = samurai::Interval; - using CellList_t = typename Mesh_t::cl_type; - using CellArray_t = samurai::CellArray; - - struct Data_t - { - size_t level; - inter_t interval; - xt::xtensor_fixed> indices; - bool given; - }; - - boost::mpi::communicator world; - - // For debug - std::ofstream logs; - logs.open("log_" + std::to_string(_rank) + ".dat", std::ofstream::app); - logs << "# New load balancing (load_balancing_sfc)" << std::endl; - - // SFC 1D key for cells - std::map sfc_map; - - logs << fmt::format("\t\t> Computing SFC ({}) 1D indices (interval) ... ", _sfc.getName() ) << std::endl; - - [[maybe_unused]] size_t ninterval = 0; - samurai::for_each_interval(mesh, [&]( std::size_t level, const auto & inter, const auto & index ) { - // get Logical coordinate or first cell - xt::xtensor_fixed> icell; - - // first element of interval - icell(0) = inter.start; - for (int idim = 0; idim < dim - 1; ++idim) - { - icell(idim + 1) = index(idim); - } - - // convert logical coordinate to max level logical coordinates - for (int idim = 0; idim < dim; ++idim) - { - icell(idim) = icell(idim) << (mesh.max_level() - level ); // +1 - } - - // this is where things can get nasty, we expect indices to be positive values !! - xt::xtensor_fixed> ijk; - for (size_t idim = 0; idim < dim; ++idim) - { - ijk(idim) = static_cast( icell(idim) ); - } - - sfc_map[_sfc.template getKey(ijk)] = {level, inter, index, false}; - - ninterval++; - }); - - assert(ninterval == sfc_map.size()); - - // Key boundaries of current process - unused for now - SFC_key_t interval[ 2 ] = { sfc_map.begin()->first, sfc_map.rbegin()->first }; - logs << "Boundaries [" << interval[ 0 ] << ", " << interval[ 1 ] << "]" << std::endl; - - std::vector load_interval; - int my_load_i = static_cast(samurai::cmptLoad(mesh)); - boost::mpi::all_gather(world, my_load_i, load_interval); - - // compute load to transfer to neighbour rank-1, rank+1 - int neighbour_rank_prev = -1, neighbour_rank_next = -1; - int transfer_load_prev = 0, transfer_load_next = 0; - - // define neighbour processes for load-balancing, not geometrical neighbour ! - if (_rank > 0) - { - neighbour_rank_prev = _rank - 1; - // transfer TRANSFER_PERCENT % max of difference - transfer_load_prev = -static_cast((my_load_i - load_interval[static_cast(neighbour_rank_prev)]) - * TRANSFER_PERCENT); - } - - if (_rank < _ndomains - 1) - { - neighbour_rank_next = _rank + 1; - // transfer TRANSFER_PERCENT % max of difference - transfer_load_next = -static_cast((my_load_i - load_interval[static_cast(neighbour_rank_next)]) - * TRANSFER_PERCENT); - } - - logs << "Neighbour prev : " << neighbour_rank_prev << ", transfer of loads : " << transfer_load_prev << std::endl; - logs << "Neighbour next : " << neighbour_rank_next << ", transfer of loads : " << transfer_load_next << std::endl; - - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Data transfer between processes ------------------------------------------------------------------ */ - /* ---------------------------------------------------------------------------------------------------------- */ - - CellList_t new_cl; // this will contains the final mesh of the current process - - // need send data to prev neighbour - if (neighbour_rank_prev >= 0 && transfer_load_prev != 0) - { - if (transfer_load_prev < 0) - { - CellList_t cl_to_send; - - // give n-smallest morton keys to prev neighbour - for (auto iter = sfc_map.begin(); iter != sfc_map.end(); ++iter) - { - if (transfer_load_prev < 0 && my_load_i > 0) - { - cl_to_send[iter->second.level][iter->second.indices].add_interval(iter->second.interval); - iter->second.given = true; // flag "interval" has been sent - my_load_i -= 1; - transfer_load_prev += 1; - } - else - { - break; - } - } - - CellArray_t ca_to_send = {cl_to_send, false}; - - logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_prev << std::endl; - - world.send(neighbour_rank_prev, 42, ca_to_send); - - // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); - // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; - - // mesh.remove( ca_to_send ); - } - else - { - // need recv - CellArray_t ca_to_rcv; - world.recv(neighbour_rank_prev, 42, ca_to_rcv); - - logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_prev << std::endl; - - // mesh.merge( ca_to_rcv ); - // add to CL what we just receive - samurai::for_each_interval(ca_to_rcv, - [&](std::size_t level, const auto& interval, const auto& index) - { - new_cl[level][index].add_interval(interval); - }); - } - } - - if (neighbour_rank_next > 0 && transfer_load_next != 0) - { - if (transfer_load_next < 0) - { - CellList_t cl_to_send; - - // give n-smallest morton keys to prev neighbour - for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) - { - if (transfer_load_next < 0 && my_load_i > 0) - { - cl_to_send[iter->second.level][iter->second.indices].add_interval(iter->second.interval); - iter->second.given = true; - my_load_i -= 1; - transfer_load_next += 1; - } - else - { - break; - } - } - - CellArray_t ca_to_send = {cl_to_send, false}; - - logs << "\t> Sending nbCells {" << ca_to_send.nb_cells() << "} to process : " << neighbour_rank_next << std::endl; - - world.send(neighbour_rank_next, 42, ca_to_send); - - // auto new_mesh = samurai::load_balance::remove( mesh, ca_to_send ); - // logs << "\t> New mesh : " << new_mesh.nb_cells() << std::endl; - - // mesh.remove( ca_to_send ); - } - else - { - // need recv - CellArray_t ca_to_rcv; - world.recv(neighbour_rank_next, 42, ca_to_rcv); - - logs << "\t> Receiving nbCells {" << ca_to_rcv.nb_cells() << "} from process : " << neighbour_rank_next << std::endl; - - // mesh.merge( ca_to_rcv ); - // add to CL what we just receive - samurai::for_each_interval(ca_to_rcv, - [&](std::size_t level, const auto& interval, const auto& index) - { - new_cl[level][index].add_interval(interval); - }); - } - } - - // last loop over the map to add what wasn't given to another process - for (auto iter = sfc_map.rbegin(); iter != sfc_map.rend(); ++iter) - { - if (!iter->second.given) - { - new_cl[iter->second.level][iter->second.indices].add_interval(iter->second.interval); - } - } - - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Construct new mesh for current process ----------------------------------------------------------- */ - /* ---------------------------------------------------------------------------------------------------------- */ - - Mesh_t new_mesh( new_cl, mesh ); - - return new_mesh; - } }; \ No newline at end of file From ff2f5d410120bdbd60bbba44100edff96d81cfd9 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 6 Jun 2024 16:03:15 +0200 Subject: [PATCH 115/170] update reordoring to use update_mesh --- include/samurai/boundary.hpp | 3 +++ include/samurai/load_balancing.hpp | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/samurai/boundary.hpp b/include/samurai/boundary.hpp index 3e0fcdcb7..e585ffe38 100644 --- a/include/samurai/boundary.hpp +++ b/include/samurai/boundary.hpp @@ -10,6 +10,9 @@ namespace samurai using mesh_id_t = typename Mesh::mesh_id_t; auto& cells = mesh[mesh_id_t::cells][level]; + // auto& domain = mesh.domain(); + // REBASE FIXME : I don't know if we need to override domain + auto& domain = mesh.subdomain(); auto max_level = domain.level(); // domain.level();//mesh[mesh_id_t::cells].max_level(); auto one_interval = layer_width << (max_level - level); diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index b7343d68a..0fb4c16b6 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -598,10 +598,16 @@ namespace samurai void reordering( Mesh_t & mesh, Field_t& field, Fields&... kw ) { // new reordered mesh on current process + MPI exchange with others - auto new_mesh = static_cast(this)->reordering_impl( mesh ); + // auto new_mesh = static_cast(this)->reordering_impl( mesh ); + logs << "\t> Computing reordering flags ... " << std::endl; + auto flags = static_cast(this)->reordering_impl( mesh ); + + logs << "\t> Update mesh based on flags ... " << std::endl; + auto new_mesh = update_mesh( mesh, flags ); // update each physical field on the new reordered mesh SAMURAI_TRACE("[Reordering::load_balance]::Updating fields ... "); + logs << "\t> Update fields based on flags ... " << std::endl; update_fields( new_mesh, field, kw... ); // swap mesh reference. @@ -674,17 +680,25 @@ namespace samurai world.recv( iproc, 17, req_recv[ static_cast( iproc ) ] ); } + for( int iproc=0; iproc( iproc ) ], req_recv[ static_cast( iproc ) ] ) << std::endl;; + } + // actual data echange between processes that need to exchange data for(int iproc=0; iproc( iproc ) ] == 1 ){ CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; + world.send( iproc, 17, to_send ); + + logs << fmt::format("\t> Sending to # {}", iproc ) << std::endl; } if( req_recv[ static_cast( iproc ) ] == 1 ) { CellArray_t to_rcv; + logs << fmt::format("\t> Recving from # {}", iproc ) << std::endl; world.recv( iproc, 17, to_rcv ); samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ @@ -692,7 +706,7 @@ namespace samurai }); } } - + Mesh_t new_mesh( new_cl, mesh ); return new_mesh; From 024256d0c544d301446a5ddefc94b33b653954d3 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 6 Jun 2024 16:04:14 +0200 Subject: [PATCH 116/170] update reordering --- include/samurai/load_balancing_sfc.hpp | 114 +++++++++++++------------ 1 file changed, 58 insertions(+), 56 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 6a530b8dd..04ed680e7 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -42,7 +42,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer - Mesh_t reordering_impl( Mesh_t & mesh ) { + auto reordering_impl( Mesh_t & mesh ) { using CellList_t = typename Mesh_t::cl_type; using CellArray_t = samurai::CellArray; @@ -71,7 +71,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer= 0 ); + assert( cell.indices( idim ) >= 0 ); ijk( idim ) = static_cast( cell.indices( idim ) ) << ( mesh.max_level() - cell.level ); } @@ -142,74 +142,76 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Communication for exchange required with rank : ["; - for( const auto & it : comm ) - logs << it.first << ","; - logs << "]" << std::endl; + // logs << "\t\t> Communication for exchange required with rank : ["; + // for( const auto & it : comm ) + // logs << it.first << ","; + // logs << "]" << std::endl; - // /* ---------------------------------------------------------------------------------------------------------- */ - // /* ------- Data transfer between processes ------------------------------------------------------------------ */ - // /* ---------------------------------------------------------------------------------------------------------- */ + // // /* ---------------------------------------------------------------------------------------------------------- */ + // // /* ------- Data transfer between processes ------------------------------------------------------------------ */ + // // /* ---------------------------------------------------------------------------------------------------------- */ - CellList_t new_cl; - std::vector payload( static_cast( world.size() ) ); + // CellList_t new_cl; + // std::vector payload( static_cast( world.size() ) ); - samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ + // samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ - if( flags[ cell ] == world.rank() ){ - if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - }else{ - if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - } - - }); - - // FIXME: this part involve a lot of communication since each process will communicate with all processes. - // This should be improved for better scalability. - for( int iproc=0; iproc( iproc ) ], false }; - world.send( iproc, 17, to_send ); - } + // if( flags[ cell ] == world.rank() ){ + // if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + // }else{ + // if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } + // if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + // } + + // }); + + // // FIXME: this part involve a lot of communication since each process will communicate with all processes. + // // This should be improved for better scalability. + // for( int iproc=0; iproc( iproc ) ], false }; + // world.send( iproc, 17, to_send ); + // } - } + // } - for( int iproc=0; iproc From 94cc599ccc2010ecbfa1ff29ba35495a16211762 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 7 Jun 2024 09:37:19 +0200 Subject: [PATCH 117/170] clean code - use of update_mesh in reordering --- include/samurai/load_balancing_sfc.hpp | 71 -------------------------- 1 file changed, 71 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 04ed680e7..60ef23a4a 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -43,8 +43,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer auto reordering_impl( Mesh_t & mesh ) { - using CellList_t = typename Mesh_t::cl_type; - using CellArray_t = samurai::CellArray; boost::mpi::communicator world; @@ -142,75 +140,6 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Communication for exchange required with rank : ["; - // for( const auto & it : comm ) - // logs << it.first << ","; - // logs << "]" << std::endl; - - // // /* ---------------------------------------------------------------------------------------------------------- */ - // // /* ------- Data transfer between processes ------------------------------------------------------------------ */ - // // /* ---------------------------------------------------------------------------------------------------------- */ - - // CellList_t new_cl; - // std::vector payload( static_cast( world.size() ) ); - - // samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ){ - - // if( flags[ cell ] == world.rank() ){ - // if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - // }else{ - // if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - // if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - // } - - // }); - - // // FIXME: this part involve a lot of communication since each process will communicate with all processes. - // // This should be improved for better scalability. - // for( int iproc=0; iproc( iproc ) ], false }; - // world.send( iproc, 17, to_send ); - // } - - // } - - // for( int iproc=0; iproc Date: Fri, 7 Jun 2024 09:38:13 +0200 Subject: [PATCH 118/170] remove useless --- include/samurai/load_balancing_diffusion_cell.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 0eacacada..3a3f5e6cd 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -54,7 +54,6 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer Date: Fri, 7 Jun 2024 09:40:42 +0200 Subject: [PATCH 119/170] update timer, add ncalls --- include/samurai/timers.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/samurai/timers.hpp b/include/samurai/timers.hpp index 90b764e5a..bdd4a889f 100644 --- a/include/samurai/timers.hpp +++ b/include/samurai/timers.hpp @@ -29,6 +29,10 @@ namespace samurai #endif uint32_t ntimes; }; +struct Timer{ + double start, elapsed; + uint32_t ntimes; +}; class Timers { From 6d8a13d1f535db89c814276a8d5a1ea86ffee149 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 10 Jun 2024 15:14:44 +0200 Subject: [PATCH 120/170] remove petsc from linear conveciton - unused --- demos/FiniteVolume/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/demos/FiniteVolume/CMakeLists.txt b/demos/FiniteVolume/CMakeLists.txt index e1d8ba6c0..1418a705b 100644 --- a/demos/FiniteVolume/CMakeLists.txt +++ b/demos/FiniteVolume/CMakeLists.txt @@ -27,6 +27,9 @@ if(PETSC_FOUND) target_link_libraries(manual_block_matrix_assembly samurai CLI11::CLI11 ${PETSC_LINK_LIBRARIES} ${MPI_LIBRARIES}) endif() +add_executable(finite-volume-linear-convection linear_convection.cpp) +target_link_libraries(finite-volume-linear-convection samurai CLI11::CLI11 ${MPI_LIBRARIES}) + add_executable(finite-volume-amr-burgers-hat AMR_Burgers_Hat.cpp) target_link_libraries(finite-volume-amr-burgers-hat samurai CLI11::CLI11) From 5f36c9f7be4b266d5c79562bbd9c9c605a4fbac9 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 10 Jun 2024 15:15:03 +0200 Subject: [PATCH 121/170] update linear convection with load balancing --- demos/FiniteVolume/linear_convection.cpp | 50 +++++++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index 910739f03..a352f102c 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -7,6 +7,15 @@ #include #include +#include +#include +#include +#include +#include +#include + +#include + #include namespace fs = std::filesystem; @@ -34,6 +43,8 @@ int main(int argc, char* argv[]) { auto& app = samurai::initialize("Finite volume example for the linear convection equation", argc, argv); + Timers myTimers; + static constexpr std::size_t dim = 2; using Config = samurai::MRConfig; using Box = samurai::Box; @@ -48,6 +59,7 @@ int main(int argc, char* argv[]) // Simulation parameters double left_box = -1; double right_box = 1; +>>>>>>> 7f2d425 (update linear convection with load balancing) // Time integration double Tf = 3; @@ -64,6 +76,7 @@ int main(int argc, char* argv[]) fs::path path = fs::current_path(); std::string filename = "linear_convection_" + std::to_string(dim) + "D"; std::size_t nfiles = 0; + std::size_t nt_loadbalance = 10; app.add_option("--left", left_box, "The left border of the box")->capture_default_str()->group("Simulation parameters"); app.add_option("--right", right_box, "The right border of the box")->capture_default_str()->group("Simulation parameters"); @@ -72,6 +85,7 @@ int main(int argc, char* argv[]) app.add_option("--cfl", cfl, "The CFL")->capture_default_str()->group("Simulation parameters"); app.add_option("--min-level", min_level, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); app.add_option("--max-level", max_level, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--nt-loadbalance", nt_loadbalance, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); app.add_option("--mr-eps", mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") ->capture_default_str() ->group("Multiresolution"); @@ -96,7 +110,7 @@ int main(int argc, char* argv[]) box_corner2.fill(right_box); Box box(box_corner1, box_corner2); std::array periodic; - periodic.fill(true); + periodic.fill(false); samurai::MRMesh mesh{box, min_level, max_level, periodic}; // Initial solution @@ -109,14 +123,17 @@ int main(int argc, char* argv[]) const auto& x = coords(0); return (x >= -0.8 && x <= -0.3) ? 1. : 0.; } - else + else if ( dim == 2 ) { const auto& x = coords(0); const auto& y = coords(1); return (x >= -0.8 && x <= -0.3 && y >= 0.3 && y <= 0.8) ? 1. : 0.; } + }); + // samurai::make_bc>(u, 0.); + auto unp1 = samurai::make_field<1>("unp1", mesh); // Intermediary fields for the RK3 scheme auto u1 = samurai::make_field<1>("u1", mesh); @@ -129,12 +146,16 @@ int main(int argc, char* argv[]) // Convection operator samurai::VelocityVector velocity; velocity.fill(1); - if constexpr (dim == 2) - { - velocity(1) = -1; - } + + // origin weno5 auto conv = samurai::make_convection_weno5(velocity); + SFC_LoadBalancer_interval balancer; + // Void_LoadBalancer balancer; + // Diffusion_LoadBalancer_cell balancer; + // Diffusion_LoadBalancer_interval balancer; + // Load_balancing::Diffusion balancer; + //--------------------// // Time iteration // //--------------------// @@ -161,6 +182,14 @@ int main(int argc, char* argv[]) double t = 0; while (t != Tf) { + + if (nt % nt_loadbalance == 0 && nt > 1 ) + { + myTimers.start("load-balancing"); + balancer.load_balance(mesh, u); + myTimers.stop("load-balancing"); + } + // Move to next timestep t += dt; if (t > Tf) @@ -171,9 +200,16 @@ int main(int argc, char* argv[]) std::cout << fmt::format("iteration {}: t = {:.2f}, dt = {}", nt++, t, dt) << std::flush; // Mesh adaptation + myTimers.start("MRadaptation"); MRadaptation(mr_epsilon, mr_regularity); + myTimers.stop("MRadaptation"); + + myTimers.start("update_ghost_mr"); samurai::update_ghost_mr(u); + myTimers.stop("update_ghost_mr"); + unp1.resize(); + u1.resize(); u2.resize(); u1.fill(0); @@ -216,6 +252,8 @@ int main(int argc, char* argv[]) << std::endl; } + myTimers.print(); + samurai::finalize(); return 0; } From 158db8f22deb0bf0f9fca980d39d5b112def04b4 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 10 Jun 2024 15:27:25 +0200 Subject: [PATCH 122/170] fix field type in linear conv + return val for lb cell --- demos/FiniteVolume/linear_convection.cpp | 21 +++++++++++-------- .../samurai/load_balancing_diffusion_cell.hpp | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index a352f102c..3697d546c 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -114,9 +114,9 @@ int main(int argc, char* argv[]) samurai::MRMesh mesh{box, min_level, max_level, periodic}; // Initial solution - auto u = samurai::make_field<1>("u", + auto u = samurai::make_field("u", mesh, - [](const auto& coords) + [&](const auto& coords) { if constexpr (dim == 1) { @@ -132,12 +132,15 @@ int main(int argc, char* argv[]) }); - // samurai::make_bc>(u, 0.); - - auto unp1 = samurai::make_field<1>("unp1", mesh); + auto unp1 = samurai::make_field("unp1", mesh); // Intermediary fields for the RK3 scheme - auto u1 = samurai::make_field<1>("u1", mesh); - auto u2 = samurai::make_field<1>("u2", mesh); + auto u1 = samurai::make_field("u1", mesh); + auto u2 = samurai::make_field("u2", mesh); + + samurai::make_bc>(u, 0.); + samurai::make_bc>(unp1, 0.); + samurai::make_bc>(u1, 0.); + samurai::make_bc>(u2, 0.); unp1.fill(0); u1.fill(0); @@ -150,9 +153,9 @@ int main(int argc, char* argv[]) // origin weno5 auto conv = samurai::make_convection_weno5(velocity); - SFC_LoadBalancer_interval balancer; + // SFC_LoadBalancer_interval balancer; // Void_LoadBalancer balancer; - // Diffusion_LoadBalancer_cell balancer; + Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; // Load_balancing::Diffusion balancer; diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 3a3f5e6cd..2ce5df7cb 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -46,8 +46,10 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer - Mesh_t reordering_impl( Mesh_t & mesh ){ - return mesh; + auto reordering_impl( Mesh_t & mesh ){ + auto flags = samurai::make_field("rank", mesh); + flags.fill( _rank ); + return flags; } template From 3b697fe16e600a64fcad1f92116327ce70d0d835 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 11 Jun 2024 08:44:33 +0200 Subject: [PATCH 123/170] add timers to samurai init --- include/samurai/samurai.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/samurai/samurai.hpp b/include/samurai/samurai.hpp index f1a449d6f..6843d9b20 100644 --- a/include/samurai/samurai.hpp +++ b/include/samurai/samurai.hpp @@ -46,6 +46,7 @@ namespace samurai #ifdef SAMURAI_WITH_MPI MPI_Init(nullptr, nullptr); #endif + times::timers.start("smr::total_runtime"); } inline void finalize() From f7de172be3abc413105de75cae6bf7cffaadbf95 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 11 Jun 2024 08:45:26 +0200 Subject: [PATCH 124/170] const + fix comment --- include/samurai/load_balancing.hpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 0fb4c16b6..68c779b37 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -213,6 +213,7 @@ namespace samurai // load of current process int my_load = static_cast( cmptLoad( mesh ) ); + logs << "> Current process load : " << my_load << std::endl; // fluxes between processes std::vector fluxes(n_neighbours, 0); @@ -253,14 +254,14 @@ namespace samurai my_load += transfertLoad; } - logs << fmt::format("it {}, neighbours : ", nt) ; + logs << fmt::format("\t> it {}, neighbours : ", nt) ; for( size_t in=0; influxes : "; for( size_t in=0; in(cmptLoad(mesh)); + logs << "> Current process load : " << my_load << std::endl; + // fluxes between processes std::vector fluxes(n_neighbours, 0); @@ -349,14 +352,14 @@ namespace samurai my_load += transfertLoad; } - logs << fmt::format("it {}, neighbours : ", nt) ; + logs << fmt::format("\t> it {}, neighbours : ", nt) ; for( size_t in=0; in fluxes : "; for( size_t in=0; in New theoretical load : " << my_load << std::endl; nt ++ ; } @@ -957,7 +960,7 @@ namespace samurai * leveldiff : max level difference. For 2:1 balance, leveldiff = 1 */ template - static auto cmptInterface(Mesh_t& mesh, Mesh_t& omesh) + static auto cmptInterface(const Mesh_t& mesh, const Mesh_t& omesh) { using CellList_t = typename Mesh_t::cl_type; using CellArray_t = typename Mesh_t::ca_type; From f657530ab3a5f3f417f8ddc4cd5e95031eb58178 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 11 Jun 2024 08:45:38 +0200 Subject: [PATCH 125/170] fix warnings --- include/samurai/load_balancing_sfc.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 60ef23a4a..0e4976594 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -228,7 +228,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( world.rank() ) ] >= static_cast( start + 1 ) * dc ){ start ++; } @@ -240,7 +240,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer= ( start + 1 ) * dc ){ start ++; - start = std::min( world.size() - 1 , start ); + start = std::min( static_cast( world.size() - 1 ) , start ); logs << "\t\t> Incrementing Start @ rank " << start << ", count " << count << std::endl; } From 0342ac26d33858f9c2c44535ad53745acda97428 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 11 Jun 2024 08:46:06 +0200 Subject: [PATCH 126/170] try fix LB cells --- .../samurai/load_balancing_diffusion_cell.hpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index 2ce5df7cb..ba75aa80f 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -20,7 +20,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer fluxes = samurai::cmptFluxes( mesh, 5 ); + std::vector fluxes = samurai::cmptFluxes( mesh, 10 ); { logs << "load : " << my_load << std::endl; @@ -104,6 +103,8 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer("rank", mesh); flags.fill( world.rank() ); @@ -144,6 +140,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer mload( static_cast( world.size() ), 0 ); for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ auto cc = cell.center(); @@ -157,8 +154,6 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( world.rank() ) ]; double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; - std::vector mload( static_cast( world.size() ), 0 ); - // select the neighbour for( std::size_t ni=0; ni( ni ); winner_dist = dist; - mload[ neighbour_rank ] += 1; + // mload[ neighbour_rank ] += 1; } } From f0b02d870655c6b8924ae2161f4ff55ded0a2470 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 11 Jun 2024 08:46:25 +0200 Subject: [PATCH 127/170] fix interface prop to use flags field --- include/samurai/load_balancing_diffusion.hpp | 93 ++++++++------------ 1 file changed, 36 insertions(+), 57 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 9c048d3f9..427ab8984 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -32,16 +32,18 @@ namespace Load_balancing{ inline std::string getName() const { return "diffusion"; } template - Mesh_t reordering_impl( Mesh_t & mesh ) { - return mesh; + auto reordering_impl( Mesh_t & mesh ) { + auto flags = samurai::make_field("diffusion_flag", mesh); + flags.fill( _rank ); + + return flags; } template - Mesh_t load_balance_impl( Mesh_t & mesh ){ + auto load_balance_impl( Mesh_t & mesh ){ using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; using CellList_t = typename Mesh_t::cl_type; - using CellArray_t = typename Mesh_t::ca_type; using mesh_id_t = typename Mesh_t::mesh_id_t; using Coord_t = xt::xtensor_fixed>; @@ -53,10 +55,37 @@ namespace Load_balancing{ std::ofstream logs; logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); logs << fmt::format("> New load-balancing using {} ", getName() ) << std::endl; + + std::vector neighbourhood; + + std::vector forceNeighbour; + { + std::vector & neighbourhood_tmp = mesh.mpi_neighbourhood(); + + for( auto & neighbour : neighbourhood_tmp ){ + auto interface = samurai::cmptInterface( mesh, neighbour.mesh ); + size_t nintervals = 0; + for_each_interval(interface, [&]( [[maybe_unused]] size_t level, [[maybe_unused]] const auto & i, [[maybe_unused]] const auto & ii ){ + nintervals ++; + }); + if( nintervals > 0 ){ + forceNeighbour.emplace_back( neighbour.rank ); + neighbourhood.emplace_back( neighbour ); + } + } + + logs << "Corrected neighbours : "; + for(const auto & fn : forceNeighbour ) + logs << fn << ", "; + logs << std::endl; + } + + size_t n_neighbours = neighbourhood.size(); // compute fluxes in terms of number of intervals to transfer/receive // by default, perform 5 iterations - std::vector fluxes = samurai::cmptFluxes( mesh, 5 ); + std::vector fluxes = samurai::cmptFluxes( mesh, forceNeighbour, 5 ); + std::vector new_fluxes( fluxes ); // get loads from everyone @@ -64,9 +93,6 @@ namespace Load_balancing{ int my_load = static_cast( samurai::cmptLoad( mesh ) ); boost::mpi::all_gather( world, my_load, loads ); - std::vector & neighbourhood = mesh.mpi_neighbourhood(); - size_t n_neighbours = neighbourhood.size(); - { // some debug info logs << "load : " << my_load << std::endl; logs << "nneighbours : " << n_neighbours << std::endl; @@ -142,7 +168,7 @@ namespace Load_balancing{ if constexpr ( Mesh_t::dim == 2 ) { if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 ){ - dir_from_neighbour[0] = 0; + // dir_from_neighbour[0] = 1; dir_from_neighbour[1] = 0; } } @@ -280,54 +306,7 @@ namespace Load_balancing{ } - CellList_t new_cl; - std::vector payload( world.size() ); - - samurai::for_each_cell( mesh[mesh_id_t::cells], [&]( const auto & cell ){ - - if( flags[ cell ] == world.rank() ){ - if constexpr ( Mesh_t::dim == 1 ){ new_cl[ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 2 ){ new_cl[ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 3 ){ new_cl[ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - }else{ - if constexpr ( Mesh_t::dim == 1 ){ payload[ flags[ cell ] ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 2 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } - if constexpr ( Mesh_t::dim == 3 ){ payload[ flags[ cell ] ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } - } - - }); - - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Data transfer between processes ------------------------------------------------------------------ */ - /* ---------------------------------------------------------------------------------------------------------- */ - - for( size_t neigh_i=0; neigh_i 0 ){ // receiver - CellArray_t to_rcv; - world.recv( neighbourhood[ neigh_i ].rank, 42, to_rcv ); - - samurai::for_each_interval(to_rcv, [&](std::size_t level, const auto & interval, const auto & index ){ - new_cl[ level ][ index ].add_interval( interval ); - }); - - }else{ // sender - CellArray_t to_send = { payload[ neighbourhood[ neigh_i ].rank ], false }; - world.send( neighbourhood[ neigh_i ].rank, 42, to_send ); - - } - - } - - /* ---------------------------------------------------------------------------------------------------------- */ - /* ------- Construct new mesh for current process ----------------------------------------------------------- */ - /* ---------------------------------------------------------------------------------------------------------- */ - - Mesh_t new_mesh( new_cl, mesh ); - - return new_mesh; + return flags; } }; From 6706f961968c27546373f0deacb0b2bbfa8e4dc2 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 11 Jun 2024 08:46:40 +0200 Subject: [PATCH 128/170] fix linear conv to use times in samurai init --- demos/FiniteVolume/linear_convection.cpp | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index 3697d546c..a221bf602 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -43,8 +43,6 @@ int main(int argc, char* argv[]) { auto& app = samurai::initialize("Finite volume example for the linear convection equation", argc, argv); - Timers myTimers; - static constexpr std::size_t dim = 2; using Config = samurai::MRConfig; using Box = samurai::Box; @@ -155,9 +153,9 @@ int main(int argc, char* argv[]) // SFC_LoadBalancer_interval balancer; // Void_LoadBalancer balancer; - Diffusion_LoadBalancer_cell balancer; + // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; - // Load_balancing::Diffusion balancer; + Load_balancing::Diffusion balancer; //--------------------// // Time iteration // @@ -188,9 +186,9 @@ int main(int argc, char* argv[]) if (nt % nt_loadbalance == 0 && nt > 1 ) { - myTimers.start("load-balancing"); + samurai::times::timers.start("load-balancing"); balancer.load_balance(mesh, u); - myTimers.stop("load-balancing"); + samurai::times::timers.stop("load-balancing"); } // Move to next timestep @@ -203,15 +201,16 @@ int main(int argc, char* argv[]) std::cout << fmt::format("iteration {}: t = {:.2f}, dt = {}", nt++, t, dt) << std::flush; // Mesh adaptation - myTimers.start("MRadaptation"); + samurai::times::timers.start("MRadaptation"); MRadaptation(mr_epsilon, mr_regularity); - myTimers.stop("MRadaptation"); + samurai::times::timers.stop("MRadaptation"); - myTimers.start("update_ghost_mr"); + samurai::times::timers.start("update_ghost_mr"); samurai::update_ghost_mr(u); - myTimers.stop("update_ghost_mr"); + samurai::times::timers.stop("update_ghost_mr"); unp1.resize(); + unp1.fill(0); u1.resize(); u2.resize(); @@ -221,12 +220,13 @@ int main(int argc, char* argv[]) // unp1 = u - dt * conv(u); // TVD-RK3 (SSPRK3) + samurai::times::timers.start("RK3"); u1 = u - dt * conv(u); samurai::update_ghost_mr(u1); u2 = 3. / 4 * u + 1. / 4 * (u1 - dt * conv(u1)); samurai::update_ghost_mr(u2); unp1 = 1. / 3 * u + 2. / 3 * (u2 - dt * conv(u2)); - + samurai::times::timers.stop("RK3"); // u <-- unp1 std::swap(u.array(), unp1.array()); @@ -255,8 +255,6 @@ int main(int argc, char* argv[]) << std::endl; } - myTimers.print(); - samurai::finalize(); return 0; } From 15d03d1938edf352f89bb6521ac78f73db6f34e9 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 13:20:56 +0200 Subject: [PATCH 129/170] fix dim type + set access to derived class to logs --- include/samurai/load_balancing.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 68c779b37..acbe35367 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -434,6 +434,8 @@ namespace samurai class LoadBalancer { private: + + public: std::ofstream logs; int nloadbalancing; @@ -911,7 +913,7 @@ namespace samurai * by passing in parameters an array for example to modulate * weight according to level */ - template + template xt::xtensor_fixed> _cmpCellBarycenter(Mesh_t& mesh) { using Coord_t = xt::xtensor_fixed>; @@ -935,7 +937,7 @@ namespace samurai auto cc = cell.center(); - for (int idim = 0; idim < dim; ++idim) + for (size_t idim = 0; idim < dim; ++idim) { bary(idim) += cc(idim) * wght; } @@ -944,7 +946,7 @@ namespace samurai }); wght_tot = std::max(wght_tot, 1e-12); - for (int idim = 0; idim < dim; ++idim) + for (size_t idim = 0; idim < dim; ++idim) { bary(idim) /= wght_tot; } From 7f7f09ae1a319e6c070887b787ed1f1aa2af3226 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 13:21:07 +0200 Subject: [PATCH 130/170] fix dim type --- include/samurai/load_balancing_sfc.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 0e4976594..40e75cd2e 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -65,7 +65,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer> ijk; + xt::xtensor_fixed> ijk; for (size_t idim = 0; idim < dim; ++idim) { // FIX need shift to get only positive index @@ -169,8 +169,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer> ijk; - for (size_t idim = 0; idim < dim; ++idim) { + xt::xtensor_fixed> ijk; + for (size_t idim = 0; idim < Mesh_t::dim; ++idim) { // FIX need shift to get only positive index assert( cell.indices( 0 ) >= 0 ); @@ -229,7 +229,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( world.rank() ) ] >= static_cast( start + 1 ) * dc ){ + while( globIdx[ static_cast( world.rank() ) ] >= ( start + 1 ) * dc ){ start ++; } From bfe3217b330496826612c928b05a63bf862a3a83 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 13:22:02 +0200 Subject: [PATCH 131/170] getStencilToNeighbour as function to allow multiple stencil instead of not moving in diagonal --- include/samurai/load_balancing_diffusion.hpp | 103 ++++++++++++++----- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 427ab8984..c93ac790f 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -12,6 +12,7 @@ namespace Load_balancing{ class Diffusion : public samurai::LoadBalancer { private: + int _ndomains; int _rank; @@ -39,6 +40,41 @@ namespace Load_balancing{ return flags; } + template + std::vector getStencilToNeighbour( const Coord_t & bc_current, const Coord_t & bc_neigh ) const { + + Stencil_t dir_from_neighbour; + std::vector stencils; + + Coord_t tmp; + double n2 = 0.; + for( size_t idim = 0; idim( tmp( idim ) / 0.5 ); + + // FIXME why needed ? + if( std::abs( dir_from_neighbour( idim ) ) > 1 ) { + dir_from_neighbour( idim ) < 0 ? dir_from_neighbour( idim ) = -1 : dir_from_neighbour( idim ) = 1; + } + + if( dir_from_neighbour( idim ) != 0 ) { + Stencil_t dd; + dd.fill(0); + dd(idim) = dir_from_neighbour(idim); + stencils.emplace_back( dd ); + } + } + + return stencils; + } + template auto load_balance_impl( Mesh_t & mesh ){ @@ -52,39 +88,45 @@ namespace Load_balancing{ boost::mpi::communicator world; // For debug - std::ofstream logs; - logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + // std::ofstream logs; + // logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); logs << fmt::format("> New load-balancing using {} ", getName() ) << std::endl; - std::vector neighbourhood; - - std::vector forceNeighbour; - { - std::vector & neighbourhood_tmp = mesh.mpi_neighbourhood(); - - for( auto & neighbour : neighbourhood_tmp ){ - auto interface = samurai::cmptInterface( mesh, neighbour.mesh ); - size_t nintervals = 0; - for_each_interval(interface, [&]( [[maybe_unused]] size_t level, [[maybe_unused]] const auto & i, [[maybe_unused]] const auto & ii ){ - nintervals ++; - }); - if( nintervals > 0 ){ - forceNeighbour.emplace_back( neighbour.rank ); - neighbourhood.emplace_back( neighbour ); - } - } - - logs << "Corrected neighbours : "; - for(const auto & fn : forceNeighbour ) - logs << fn << ", "; - logs << std::endl; - } + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + + /* + * Correction of the list of neighbour process to take into account into the graph when computing + * fluxes that this load-balancing strategy does not exchange in diagonal. Might no longer be necessary + * if stencil are splitted by direction. + */ + // std::vector forceNeighbour; + // { + // std::vector & neighbourhood_tmp = mesh.mpi_neighbourhood(); + + // for( auto & neighbour : neighbourhood_tmp ){ + // auto interface = samurai::cmptInterface( mesh, neighbour.mesh ); + // size_t nintervals = 0; + // for_each_interval(interface, [&]( [[maybe_unused]] size_t level, [[maybe_unused]] const auto & i, [[maybe_unused]] const auto & ii ){ + // nintervals ++; + // }); + // if( nintervals > 0 ){ + // forceNeighbour.emplace_back( neighbour.rank ); + // neighbourhood.emplace_back( neighbour ); + // } + // } + + // logs << "Corrected neighbours : "; + // for(const auto & fn : forceNeighbour ) + // logs << fn << ", "; + // logs << std::endl; + // } size_t n_neighbours = neighbourhood.size(); // compute fluxes in terms of number of intervals to transfer/receive // by default, perform 5 iterations - std::vector fluxes = samurai::cmptFluxes( mesh, forceNeighbour, 5 ); + // std::vector fluxes = samurai::cmptFluxes( mesh, forceNeighbour, 5 ); + std::vector fluxes = samurai::cmptFluxes( mesh, 5 ); std::vector new_fluxes( fluxes ); @@ -144,6 +186,13 @@ namespace Load_balancing{ // Compute normalized direction to neighbour Stencil dir_from_neighbour; { + auto aa_ = getStencilToNeighbour( bc_current, bc_neighbour ); + + logs << fmt::format("\t> Number of stencils: {}", aa_.size()) << std::endl; + for( const auto & st : aa_ ) { + logs << fmt::format("\t\t> Stencil : ({},{})", st(0), st(1) ) << std::endl; + } + Coord_t tmp; double n2 = 0.; for( size_t idim = 0; idim Date: Tue, 11 Jun 2024 13:22:16 +0200 Subject: [PATCH 132/170] fix dim type --- .../load_balancing_diffusion_interval.hpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index aeaf058aa..4d53cfa26 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -86,7 +86,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer; + using CellArray_t = samurai::CellArray; using mesh_id_t = typename Mesh_t::mesh_id_t; boost::mpi::communicator world; @@ -157,7 +157,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer( mesh, neighbourhood[ requester ].mesh ); + auto interface = samurai::cmptInterface( mesh, neighbourhood[ requester ].mesh ); { // check emptyness of interface, if it is empty, then set fluxes for this neighbour to 0 size_t nelement = 0; @@ -285,7 +285,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer; + using CellArray_t = samurai::CellArray; using mesh_id_t = typename Mesh_t::mesh_id_t; boost::mpi::communicator world; @@ -321,11 +321,11 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer( mesh ); + auto interface = samurai::_computeCartesianInterface( mesh ); // compute some point of reference in mesh and interval-based interface // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); - Coord_t barycenter = samurai::_cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = samurai::_cmpCellBarycenter( mesh[ mesh_id_t::cells ] ); logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; std::vector invloads; @@ -341,7 +341,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer( interface[ nbi ] ); - barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); + barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); // debug // auto s_ = fmt::format( "Barycenter neighbour : ({}, {})", @@ -354,7 +354,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer> indices; + xt::xtensor_fixed> indices; int rank; }; @@ -372,7 +372,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer::max(); - double winner_dist = samurai::getDistance( mid_point, barycenter ) * invloads[ _rank ]; + double winner_dist = samurai::getDistance( mid_point, barycenter ) * invloads[ _rank ]; // select the neighbour for( size_t ni=0; ni( mid_point, barycenter_interface_neighbours[ ni ] ), // distance_inf( mid_point, barycenter_neighbours[ ni ] ) ); - double dist = samurai::getDistance( mid_point, barycenter_neighbours[ ni ] ) * invloads[ neighbour_rank ] ; - // double dist = samurai::distance_inf( mid_point, barycenter_interface_neighbours[ ni ] ); + double dist = samurai::getDistance( mid_point, barycenter_neighbours[ ni ] ) * invloads[ neighbour_rank ] ; + // double dist = samurai::distance_inf( mid_point, barycenter_interface_neighbours[ ni ] ); if( dist < winner_dist ){ winner_id = ni; @@ -441,7 +441,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer 0 ) { // receive data - samurai::CellArray to_rcv; + samurai::CellArray to_rcv; world.recv( neighbourhood[ ni ].rank, 42, to_rcv ); mesh.merge( to_rcv ); }else{ // send data to From 98a2cb74247d32434a0ca054b1ca3b2165bc08d6 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 14:03:41 +0200 Subject: [PATCH 133/170] fix dim --- include/samurai/load_balancing.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index acbe35367..2d43dcbd4 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -122,11 +122,11 @@ namespace samurai * Compute distance base on different norm. */ - template + template static inline double distance_l2(const Coord_t& d1, const Coord_t& d2) { double dist = 0.; - for (size_t idim = 0; idim < static_cast(dim); ++idim) + for (size_t idim = 0; idim < dim; ++idim) { double d = d1(idim) - d2(idim); dist += d * d; @@ -134,22 +134,22 @@ namespace samurai return std::sqrt(dist); } - template + template static inline double distance_inf(const Coord_t& d1, const Coord_t& d2) { double dist = 0.; - for (size_t idim = 0; idim < static_cast(dim); ++idim) + for (size_t idim = 0; idim + template static inline double distance_l1(const Coord_t& d1, const Coord_t& d2) { double dist = 0.; - for (size_t idim = 0; idim < static_cast(dim); ++idim) + for (size_t idim = 0; idim < dim; ++idim) { dist += std::abs(d1(idim) - d2(idim)); } @@ -389,7 +389,7 @@ namespace samurai * What should be m ? m' ? Let's G = 1. * **/ - template + template static inline double gravity(const Coord_t& d1, const Coord_t& d2) { double dist = distance_l2(d1, d2); @@ -408,7 +408,7 @@ namespace samurai } // we are using Cell_t to allow ponderation using level; - template + template inline constexpr double getDistance(const Coord_t& cc, const Coord_t& d) { static_assert(dim == 2 || dim == 3); @@ -864,7 +864,7 @@ namespace samurai /** * Precompute direction to element in all direction face + diagonals: 26 in 3D, 8 in 2D */ - template + template constexpr auto getDirectionFaceAndDiag() { using base_stencil = xt::xtensor_fixed>; From 41e64e19e66d36b18e0c6eff304d31df7ae0c9f1 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 14:10:17 +0200 Subject: [PATCH 134/170] fix warning --- include/samurai/load_balancing_diffusion.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index c93ac790f..060cdac84 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -280,7 +280,7 @@ namespace Load_balancing{ // propagate until full-fill neighbour { - int nbElementGiven = 1; // validate the while condition on starter + size_t nbElementGiven = 1; // validate the while condition on starter logs << fmt::format("\t\t\t> Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; @@ -343,7 +343,7 @@ namespace Load_balancing{ // interface = { cl_given, false }; - new_fluxes[ neighbour_local_id ] += nbElementGiven; + new_fluxes[ neighbour_local_id ] += static_cast( nbElementGiven ); offset ++; } From 557a5aef9514a305f5aae7df8a795b9ae3ca0ada Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 14:10:28 +0200 Subject: [PATCH 135/170] fix warning --- include/samurai/load_balancing_diffusion_interval.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index 4d53cfa26..822eba13a 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -3,7 +3,7 @@ #include #include "load_balancing.hpp" -template +template class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer> { using Coord_t = xt::xtensor_fixed>; @@ -33,7 +33,7 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer WARNING: Key conflict for std::map !" << std::endl; } - repartition.insert ( std::pair( winner_dist, Data { level, interval, index, static_cast( winner_id ) } ) ); + repartition.insert ( std::pair( winner_dist, Data { level, interval, index, winner_id } ) ); } From 4fe5efc1fb2cbf1dcd89e02dcbcd64c1a165261b Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 14:10:48 +0200 Subject: [PATCH 136/170] fix warning --- include/samurai/load_balancing_diffusion_cell.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_diffusion_cell.hpp index ba75aa80f..1fdc40ce7 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_diffusion_cell.hpp @@ -3,7 +3,7 @@ #include #include "load_balancing.hpp" -template +template class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer> { using Coord_t = xt::xtensor_fixed>; From c33a5a4e8aa742c6e642e39fca5f1b088ff946e7 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 14:11:08 +0200 Subject: [PATCH 137/170] add timer for init --- demos/FiniteVolume/linear_convection.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index a221bf602..9132c18fc 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -109,6 +109,8 @@ int main(int argc, char* argv[]) Box box(box_corner1, box_corner2); std::array periodic; periodic.fill(false); + + samurai::times::timers.start("init"); samurai::MRMesh mesh{box, min_level, max_level, periodic}; // Initial solution @@ -129,6 +131,7 @@ int main(int argc, char* argv[]) } }); + samurai::times::timers.stop("init"); auto unp1 = samurai::make_field("unp1", mesh); // Intermediary fields for the RK3 scheme @@ -170,7 +173,10 @@ int main(int argc, char* argv[]) } auto MRadaptation = samurai::make_MRAdapt(u); + + samurai::times::timers.start("MRadaptation"); MRadaptation(mr_epsilon, mr_regularity); + samurai::times::timers.stop("MRadaptation"); double dt_save = nfiles == 0 ? dt : Tf / static_cast(nfiles); std::size_t nsave = 0, nt = 0; @@ -198,7 +204,7 @@ int main(int argc, char* argv[]) dt += Tf - t; t = Tf; } - std::cout << fmt::format("iteration {}: t = {:.2f}, dt = {}", nt++, t, dt) << std::flush; + std::cout << fmt::format("iteration {}: t = {:.2f}, dt = {}", nt++, t, dt) << std::flush << std::endl; // Mesh adaptation samurai::times::timers.start("MRadaptation"); @@ -244,7 +250,6 @@ int main(int argc, char* argv[]) } } - std::cout << std::endl; } if constexpr (dim == 1) From 16ebe250ffd608099852fd5552b6d8613121a740 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 15:43:12 +0200 Subject: [PATCH 138/170] fix warning --- include/samurai/load_balancing_sfc.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 40e75cd2e..7d84063c2 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -17,6 +17,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer>::logs; + SFC_LoadBalancer_interval() { #ifdef SAMURAI_WITH_MPI @@ -47,8 +49,8 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer Incrementing Start @ rank " << start << ", count " << count << std::endl; } - flags[ it.second ] = start; + flags[ it.second ] = static_cast( start ); count ++; } From 158b9df8b1dfe004cb0e0cd806675744d1721494 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 23:00:48 +0200 Subject: [PATCH 139/170] fix diffusion by splitting dirs --- include/samurai/load_balancing_diffusion.hpp | 269 ++++++++++--------- 1 file changed, 137 insertions(+), 132 deletions(-) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 060cdac84..4ab98539a 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -184,173 +184,178 @@ namespace Load_balancing{ Coord_t bc_neighbour = samurai::_cmpCellBarycenter( neighbourhood[ neighbour_local_id ].mesh[ mesh_id_t::cells ] ); // Compute normalized direction to neighbour - Stencil dir_from_neighbour; - { - auto aa_ = getStencilToNeighbour( bc_current, bc_neighbour ); - - logs << fmt::format("\t> Number of stencils: {}", aa_.size()) << std::endl; - for( const auto & st : aa_ ) { - logs << fmt::format("\t\t> Stencil : ({},{})", st(0), st(1) ) << std::endl; + // Stencil dir_from_neighbour; + std::vector dirs = getStencilToNeighbour( bc_current, bc_neighbour ); + // { + // auto aa_ = getStencilToNeighbour( bc_current, bc_neighbour ); + + // logs << fmt::format("\t> Number of stencils: {}", aa_.size()) << std::endl; + // for( const auto & st : aa_ ) { + // logs << fmt::format("\t\t> Stencil : ({},{})", st(0), st(1) ) << std::endl; + // } + + // Coord_t tmp; + // double n2 = 0.; + // for( size_t idim = 0; idim( tmp( idim ) / 0.5 ); + + // // FIXME why needed ? + // if( std::abs( dir_from_neighbour( idim ) ) > 1 ) { + // dir_from_neighbour( idim ) < 0 ? dir_from_neighbour( idim ) = -1 : dir_from_neighbour( idim ) = 1; + // } + // } + + // // Avoid diagonals exchange, and emphaze x-axis. Maybe two phases propagation in case of diagonal ? + // // i.e.: if (1, 1) -> (1, 0) then (0, 1) ? + + // if constexpr ( Mesh_t::dim == 2 ) { + // if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 ){ + // // dir_from_neighbour[0] = 1; + // dir_from_neighbour[1] = 0; + // } + // } + + // if constexpr ( Mesh_t::dim == 3 ) { + // if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 && std::abs(dir_from_neighbour[2]) == 1){ + // dir_from_neighbour[1] = 0; + // dir_from_neighbour[2] = 0; + // } + // } + + // logs << fmt::format("\t\t> (corrected) stencil for this neighbour # {} :", neighbourhood[ neighbour_local_id ].rank); + // for(size_t idim=0; idim( mesh, neighbourhood[ neighbour_local_id ].mesh, dir_from_neighbour ); + + bool empty = false; + { + size_t iii = 0; + samurai::for_each_interval( interface, [&]( [[maybe_unused]] size_t level, [[maybe_unused]] const auto & i, [[maybe_unused]] const auto & ii ){ + iii ++; + }); + if( iii == 0 ) empty = true; } - Coord_t tmp; - double n2 = 0.; - for( size_t idim = 0; idim Skipping neighbour, empty interface ! " << std::endl; + continue; } - n2 = std::sqrt( n2 ); - - for( size_t idim = 0; idim( tmp( idim ) / 0.5 ); + { + int nCellsAtInterfaceGiven = 0, nCellsAtInterface = 0; + for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { + + auto intersect = samurai::intersection( interface[ interface.min_level() ], mesh[ mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! + intersect( [&]( [[maybe_unused]] const auto & interval, [[maybe_unused]] const auto & index ){ + for(size_t ii=0; ii 1 ) { - dir_from_neighbour( idim ) < 0 ? dir_from_neighbour( idim ) = -1 : dir_from_neighbour( idim ) = 1; - } - } - - // Avoid diagonals exchange, and emphaze x-axis. Maybe two phases propagation in case of diagonal ? - // i.e.: if (1, 1) -> (1, 0) then (0, 1) ? + }); - if constexpr ( Mesh_t::dim == 2 ) { - if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 ){ - // dir_from_neighbour[0] = 1; - dir_from_neighbour[1] = 0; } - } - if constexpr ( Mesh_t::dim == 3 ) { - if( std::abs(dir_from_neighbour[0]) == 1 && std::abs( dir_from_neighbour[1]) == 1 && std::abs(dir_from_neighbour[2]) == 1){ - dir_from_neighbour[1] = 0; - dir_from_neighbour[2] = 0; - } - } + new_fluxes[ neighbour_local_id ] += nCellsAtInterfaceGiven; - logs << fmt::format("\t\t> (corrected) stencil for this neighbour # {} :", neighbourhood[ neighbour_local_id ].rank); - for(size_t idim=0; idim NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}", nCellsAtInterface, nCellsAtInterfaceGiven ) << std::endl; } - logs << std::endl; + - } + // propagate until full-fill neighbour + { + size_t nbElementGiven = 1; // validate the while condition on starter + + logs << fmt::format("\t\t\t> Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; - // direction from neighbour domain to current domain - auto interface = samurai::cmptInterfaceUniform( mesh, neighbourhood[ neighbour_local_id ].mesh, dir_from_neighbour ); + int offset = 1; + while( new_fluxes[ neighbour_local_id ] < 0 && nbElementGiven > 0 ){ - bool empty = false; - { - size_t iii = 0; - samurai::for_each_interval( interface, [&]( [[maybe_unused]] size_t level, [[maybe_unused]] const auto & i, [[maybe_unused]] const auto & ii ){ - iii ++; - }); - if( iii == 0 ) empty = true; - } + // intersection of interface with current mesh + size_t minLevelInInterface = mesh.max_level(); - if( empty ) { - logs << "\t> Skipping neighbour, empty interface ! " << std::endl; - continue; - } + { + auto interface_on_mesh = samurai::translate( interface[ interface.min_level() ], dir_from_neighbour * offset ); // interface is monolevel ! + for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { + size_t nIntervalAtInterface = 0; + auto intersect = samurai::intersection( interface_on_mesh, mesh[ mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! + intersect( [&]( [[maybe_unused]] const auto & interval, [[maybe_unused]] const auto & index ){ + nIntervalAtInterface += 1; + }); - { - int nCellsAtInterfaceGiven = 0, nCellsAtInterface = 0; - for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { - - auto intersect = samurai::intersection( interface[ interface.min_level() ], mesh[ mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! - intersect( [&]( [[maybe_unused]] const auto & interval, [[maybe_unused]] const auto & index ){ - for(size_t ii=0; ii 0 ) minLevelInInterface = std::min( minLevelInInterface, level ); } - nCellsAtInterface += 1; } - }); - - } - - new_fluxes[ neighbour_local_id ] += nCellsAtInterfaceGiven; - - logs << fmt::format("\t\t> NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}", nCellsAtInterface, nCellsAtInterfaceGiven ) << std::endl; - } - - // propagate until full-fill neighbour - { - size_t nbElementGiven = 1; // validate the while condition on starter - - logs << fmt::format("\t\t\t> Propagate for neighbour rank # {}", neighbourhood[ neighbour_local_id ].rank) << std::endl; - - int offset = 1; - while( new_fluxes[ neighbour_local_id ] < 0 && nbElementGiven > 0 ){ - - // intersection of interface with current mesh - size_t minLevelInInterface = mesh.max_level(); - - { - auto interface_on_mesh = samurai::translate( interface[ interface.min_level() ], dir_from_neighbour * offset ); // interface is monolevel ! - for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { - size_t nIntervalAtInterface = 0; - auto intersect = samurai::intersection( interface_on_mesh, mesh[ mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! - intersect( [&]( [[maybe_unused]] const auto & interval, [[maybe_unused]] const auto & index ){ - nIntervalAtInterface += 1; - }); - - if( nIntervalAtInterface > 0 ) minLevelInInterface = std::min( minLevelInInterface, level ); + if( minLevelInInterface != interface.min_level() ) { + logs << "\t\t\t\t> [WARNING] Interface need to be update !" << std::endl; } - } + + logs << fmt::format("\t\t\t\t> Min level in interface : {}", minLevelInInterface ) << std::endl; - if( minLevelInInterface != interface.min_level() ) { - logs << "\t\t\t\t> [WARNING] Interface need to be update !" << std::endl; - } - - logs << fmt::format("\t\t\t\t> Min level in interface : {}", minLevelInInterface ) << std::endl; + // CellList_t cl_given; - // CellList_t cl_given; + nbElementGiven = 0; - nbElementGiven = 0; + auto interface_on_mesh = translate( interface[ interface.min_level() ], dir_from_neighbour * offset ); // interface is monolevel ! + for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { - auto interface_on_mesh = translate( interface[ interface.min_level() ], dir_from_neighbour * offset ); // interface is monolevel ! - for (size_t level = mesh.min_level(); level <= mesh.max_level(); ++level) { + size_t nCellsAtInterface = 0, nCellsAtInterfaceGiven = 0; + auto intersect = intersection( interface_on_mesh, mesh[ Mesh_t::mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! - size_t nCellsAtInterface = 0, nCellsAtInterfaceGiven = 0; - auto intersect = intersection( interface_on_mesh, mesh[ Mesh_t::mesh_id_t::cells ][ level ] ).on( level ); // need handle level difference here ! + intersect( [&]( const auto & interval, const auto & index ){ - intersect( [&]( const auto & interval, const auto & index ){ + for(size_t ii=0; ii At level {}, NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}, nbElementGiven : {}", level, - nCellsAtInterface, nCellsAtInterfaceGiven, nbElementGiven ) << std::endl; + logs << fmt::format("\t\t\t\t> At level {}, NCellsAtInterface : {}, NCellsAtInterfaceGiven : {}, nbElementGiven : {}", level, + nCellsAtInterface, nCellsAtInterfaceGiven, nbElementGiven ) << std::endl; - nbElementGiven += nCellsAtInterfaceGiven; + nbElementGiven += nCellsAtInterfaceGiven; - } + } - // interface = { cl_given, false }; + // interface = { cl_given, false }; - new_fluxes[ neighbour_local_id ] += static_cast( nbElementGiven ); - offset ++; + new_fluxes[ neighbour_local_id ] += static_cast( nbElementGiven ); + offset ++; + } + } - - } - if( new_fluxes[ neighbour_local_id ] < 0 ){ - logs << fmt::format("\t> Error cannot fullfill the neighbour # {}, fluxes: {} ", neighbourhood[ neighbour_local_id ].rank, new_fluxes[ neighbour_local_id ] ) << std::endl; + if( new_fluxes[ neighbour_local_id ] < 0 ){ + logs << fmt::format("\t> Error cannot fullfill the neighbour # {}, fluxes: {} ", neighbourhood[ neighbour_local_id ].rank, new_fluxes[ neighbour_local_id ] ) << std::endl; + } + } } From cb910a7bc9461d9c10771681f7dcb1cf260fcce0 Mon Sep 17 00:00:00 2001 From: Loic Strafella Date: Tue, 11 Jun 2024 23:01:34 +0200 Subject: [PATCH 140/170] add timers --- demos/FiniteVolume/linear_convection.cpp | 28 +++++++++++++++--------- include/samurai/load_balancing.hpp | 2 +- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index 9132c18fc..057856fca 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -154,11 +154,11 @@ int main(int argc, char* argv[]) // origin weno5 auto conv = samurai::make_convection_weno5(velocity); - // SFC_LoadBalancer_interval balancer; + SFC_LoadBalancer_interval balancer; // Void_LoadBalancer balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; - Load_balancing::Diffusion balancer; + // Load_balancing::Diffusion balancer; //--------------------// // Time iteration // @@ -186,15 +186,16 @@ int main(int argc, char* argv[]) save(path, filename, u, suffix); } + samurai::times::timers.start("tloop"); double t = 0; while (t != Tf) { if (nt % nt_loadbalance == 0 && nt > 1 ) { - samurai::times::timers.start("load-balancing"); + samurai::times::timers.start("tloop.load-balancing"); balancer.load_balance(mesh, u); - samurai::times::timers.stop("load-balancing"); + samurai::times::timers.stop("tloop.load-balancing"); } // Move to next timestep @@ -204,17 +205,19 @@ int main(int argc, char* argv[]) dt += Tf - t; t = Tf; } + std::cout << fmt::format("iteration {}: t = {:.2f}, dt = {}", nt++, t, dt) << std::flush << std::endl; // Mesh adaptation - samurai::times::timers.start("MRadaptation"); + samurai::times::timers.start("tloop.MRadaptation"); MRadaptation(mr_epsilon, mr_regularity); - samurai::times::timers.stop("MRadaptation"); + samurai::times::timers.stop("tloop.MRadaptation"); - samurai::times::timers.start("update_ghost_mr"); + samurai::times::timers.start("tloop.ugm"); samurai::update_ghost_mr(u); - samurai::times::timers.stop("update_ghost_mr"); + samurai::times::timers.stop("tloop.ugm"); + samurai::times::timers.start("tloop.resize_fill"); unp1.resize(); unp1.fill(0); @@ -222,21 +225,24 @@ int main(int argc, char* argv[]) u2.resize(); u1.fill(0); u2.fill(0); + samurai::times::timers.stop("tloop.resize_fill"); // unp1 = u - dt * conv(u); // TVD-RK3 (SSPRK3) - samurai::times::timers.start("RK3"); + samurai::times::timers.start("tloop.RK3"); u1 = u - dt * conv(u); samurai::update_ghost_mr(u1); u2 = 3. / 4 * u + 1. / 4 * (u1 - dt * conv(u1)); samurai::update_ghost_mr(u2); unp1 = 1. / 3 * u + 2. / 3 * (u2 - dt * conv(u2)); - samurai::times::timers.stop("RK3"); + samurai::times::timers.stop("tloop.RK3"); + // u <-- unp1 std::swap(u.array(), unp1.array()); // Save the result + samurai::times::timers.start("tloop.io"); if (nfiles == 0 || t >= static_cast(nsave + 1) * dt_save || t == Tf) { if (nfiles != 1) @@ -249,8 +255,10 @@ int main(int argc, char* argv[]) save(path, filename, u); } } + samurai::times::timers.stop("tloop.io"); } + samurai::times::timers.stop("tloop"); if constexpr (dim == 1) { diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 2d43dcbd4..1c1ac4a5f 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -256,7 +256,7 @@ namespace samurai logs << fmt::format("\t> it {}, neighbours : ", nt) ; for( size_t in=0; influxes : "; for( size_t in=0; in Date: Wed, 12 Jun 2024 16:26:03 +0200 Subject: [PATCH 141/170] try fix loadbalancing strategy based on geo dist --- demos/FiniteVolume/linear_convection.cpp | 6 +-- include/samurai/load_balancing.hpp | 13 +++--- ...sion_cell.hpp => load_balancing_force.hpp} | 43 +++++++++++-------- 3 files changed, 35 insertions(+), 27 deletions(-) rename include/samurai/{load_balancing_diffusion_cell.hpp => load_balancing_force.hpp} (79%) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index 057856fca..15898317f 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include @@ -154,9 +154,9 @@ int main(int argc, char* argv[]) // origin weno5 auto conv = samurai::make_convection_weno5(velocity); - SFC_LoadBalancer_interval balancer; + // SFC_LoadBalancer_interval balancer; // Void_LoadBalancer balancer; - // Diffusion_LoadBalancer_cell balancer; + Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; // Load_balancing::Diffusion balancer; diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 1c1ac4a5f..7549dd8b1 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -113,7 +113,7 @@ namespace samurai None }; - static const double load_balancing_threshold = 0.03141592; // 2.5 % + static const double load_balancing_threshold = 0.01; // 0.03141592; // 2.5 % // static const std::vector load_balancing_cell_weight = { 1., 0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, // 0.0078125, 0.00390625, 0.001953125, 0.0009765625, // 0.00048828125, 0.000244140625, 0.0001220703125 }; @@ -256,7 +256,7 @@ namespace samurai logs << fmt::format("\t> it {}, neighbours : ", nt) ; for( size_t in=0; in( neighbourhood[ in ] ) ] << "], "; logs << std::endl << "\t\t>fluxes : "; for( size_t in=0; in it {}, neighbours : ", nt) ; for( size_t in=0; in( neighbourhood[ in ].rank ) ] << "], "; logs << std::endl << "\t> fluxes : "; for( size_t in=0; in + template xt::xtensor_fixed> _cmpCellBarycenter(Mesh_t& mesh) { using Coord_t = xt::xtensor_fixed>; @@ -1380,8 +1381,8 @@ namespace samurai constexpr size_t dim = Mesh_t::dim; // operation are on leaves cells only - auto meshA_ = meshA[mesh_id_t::cells]; - auto meshB_ = meshB[mesh_id_t::cells]; + auto & meshA_ = meshA[mesh_id_t::cells]; + auto & meshB_ = meshB[mesh_id_t::cells]; // get stencils for direction auto dirs = getDirection(); diff --git a/include/samurai/load_balancing_diffusion_cell.hpp b/include/samurai/load_balancing_force.hpp similarity index 79% rename from include/samurai/load_balancing_diffusion_cell.hpp rename to include/samurai/load_balancing_force.hpp index 1fdc40ce7..3465f834e 100644 --- a/include/samurai/load_balancing_diffusion_cell.hpp +++ b/include/samurai/load_balancing_force.hpp @@ -56,8 +56,6 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer fluxes = samurai::cmptFluxes( mesh, 10 ); + std::vector fluxes = samurai::cmptFluxes( mesh, 1 ); { logs << "load : " << my_load << std::endl; @@ -90,16 +88,14 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer auto interface = samurai::_computeCartesianInterface( mesh ); - std::vector ncells_interface ( interface.size(), 0 ); - for(size_t ni=0; ni ncells_interface ( n_neighbours, 0 ); + for(size_t ni=0; ni( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = samurai::_cmpCellBarycenter( mesh ); logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; // std::vector barycenter_interface_neighbours( n_neighbours ); @@ -119,7 +115,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer( interface[ nbi ] ); - barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh[ mesh_id_t::cells ] ); + barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh ); // surface or volume depending on dim sv[ nbi ] = getSurfaceOrVolume( neighbourhood[ nbi ].mesh ); @@ -136,7 +132,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer("rank", mesh); flags.fill( world.rank() ); - constexpr auto fdist = samurai::Distance_t::GRAVITY; + constexpr auto fdist = samurai::Distance_t::L2; double currentSV = getSurfaceOrVolume( mesh ); @@ -151,9 +147,12 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer::max(); // double winner_dist = samurai::getDistance( cell, barycenter ) / loads[ world.rank() ]; - double coeff_current = currentSV / loads[ static_cast( world.rank() ) ]; + // double coeff_current = currentSV / loads[ static_cast( world.rank() ) ]; + double coeff_current = 1.; // loads[ static_cast( world.rank() ) ]; double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; + // logs << fmt::format("\t\t> Cell : ({},{}), current mesh dist : {}", cc(0), cc(1), winner_dist ) << std::endl; + // select the neighbour for( std::size_t ni=0; ni( mid_point, barycenter_neighbours[ ni ] ) ); // double dist = samurai::getDistance( cell, barycenter_interface_neighbours[ ni ] ) / loads[ neighbour_rank ]; - double coeff = sv[ ni ] / ( loads[ neighbour_rank ] + mload[ neighbour_rank ] ) ; // / sv[ ni ]; + // double coeff = sv[ ni ] / ( loads[ neighbour_rank ] ) ; // / sv[ ni ]; + double coeff = 1.; // loads[ neighbour_rank ] ; // mload[ neighbour_rank ]; double dist = samurai::getDistance( cell.center(), barycenter_neighbours[ ni ] ) * coeff; + // logs << fmt::format("\t\t\t> Dist to neighbour {} : {}", neighbour_rank, dist ) << std::endl; + if( dist < winner_dist && ncells_interface[ ni ] > 0 ){ winner_id = static_cast( ni ); winner_dist = dist; - // mload[ neighbour_rank ] += 1; + mload[ neighbour_rank ] += 1; + + // if( mload[ neighbour_rank ] >= ( - fluxes[ ni ] ) ) fluxes[ ni ] = 0; } } + // logs << fmt::format("\t\t\t> Cell given to process #{}", neighbourhood[ static_cast( winner_id ) ].rank ) << std::endl; + if( winner_id >= 0 ){ + assert( winner_id < neighbourhood.size() ); flags[ cell ] = neighbourhood[ static_cast( winner_id ) ].rank; } From f91c07fb36aa1a262296951e64ec712e9e791dce Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 13 Jun 2024 16:19:12 +0200 Subject: [PATCH 142/170] fix missing include for tests + add celLExists + add unit test --- demos/FiniteVolume/linear_convection.cpp | 18 +++++-- include/samurai/load_balancing.hpp | 22 +++++++++ tests/test_loadbalancing.cpp | 60 +++++++++++++++++++++++- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index 15898317f..7723698b1 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -13,9 +13,14 @@ #include #include #include +#include #include +#ifdef WITH_STATS +#include "samurai/statistics.hpp" +#endif + #include namespace fs = std::filesystem; @@ -53,6 +58,7 @@ int main(int argc, char* argv[]) //--------------------// // Program parameters // //--------------------// + boost::mpi::communicator world; // Simulation parameters double left_box = -1; @@ -155,8 +161,9 @@ int main(int argc, char* argv[]) auto conv = samurai::make_convection_weno5(velocity); // SFC_LoadBalancer_interval balancer; + Load_balancing::Life balancer; // Void_LoadBalancer balancer; - Diffusion_LoadBalancer_cell balancer; + // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; // Load_balancing::Diffusion balancer; @@ -193,9 +200,14 @@ int main(int argc, char* argv[]) if (nt % nt_loadbalance == 0 && nt > 1 ) { - samurai::times::timers.start("tloop.load-balancing"); + samurai::times::timers.start("tloop.lb:"+balancer.getName()); balancer.load_balance(mesh, u); - samurai::times::timers.stop("tloop.load-balancing"); + samurai::times::timers.stop("tloop.lb:"+balancer.getName()); + + // std::string _stats = fmt::format("statistics_process_{}", world.rank()); + // samurai::Statistics s(_stats); + // s("statistics", mesh); + } // Move to next timestep diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 7549dd8b1..db939fe31 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -3,9 +3,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -430,6 +432,26 @@ namespace samurai } } + template + bool cellExists( const Mesh_t & mesh, Id_t mesh_id, std::size_t level, const Coord_t & lo ){ + using CellList_t = typename Mesh_t::cl_type; + using CellArray_t = typename Mesh_t::ca_type; + + CellList_t tmp; + if constexpr( Mesh_t::dim == 2 ) { tmp[ level ][ { lo( 1 ) } ].add_point( lo( 0 ) ); } + if constexpr( Mesh_t::dim == 3 ) { tmp[ level ][ { lo( 1 ), lo( 2 ) } ].add_point( lo( 0 ) ); } + + CellArray_t tmp_ = { tmp, false }; + auto set = intersection( mesh[ mesh_id ][level], tmp_[level]); + + size_t ninterval = 0; + set( [&]([[maybe_unused]]const auto& i, [[maybe_unused]]const auto& index) { + ninterval ++ ; + }); + + return ninterval > 0 ? true : false ; + } + template class LoadBalancer { diff --git a/tests/test_loadbalancing.cpp b/tests/test_loadbalancing.cpp index 5d2a52bcc..bec339199 100644 --- a/tests/test_loadbalancing.cpp +++ b/tests/test_loadbalancing.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #ifdef SAMURAI_WITH_MPI #include @@ -83,11 +85,65 @@ namespace samurai { auto interface = cmptInterface( mesh_a, mesh_b ); - std::cerr << interface << std::endl; - ASSERT_EQ( interface.nb_cells(), 10 ); } + TEST(loadBalance, cellExists){ + constexpr int dim = 2; + + using Config = samurai::MRConfig; + using Mesh_t = samurai::MRMesh; + + samurai::CellList cl_a, cl_b; + + { + cl_a[0][{0}].add_interval({0, 4}); // level 0 + + cl_a[1][{2}].add_interval({0, 2}); // level 1 + cl_a[1][{3}].add_interval({0, 2}); // level 1 + cl_a[1][{2}].add_interval({6, 8}); // level 1 + cl_a[1][{3}].add_interval({6, 8}); // level 1 + + cl_a[1][{2}].add_interval({2, 6}); // level 1 + cl_a[2][{6}].add_interval({4, 6}); // level 2 + cl_a[2][{6}].add_interval({10, 12}); // level 2 + } + + Mesh_t mesh ( cl_a, 0, 2 ); + + { + xt::xtensor_fixed> ij = {{3, 2}}; + ASSERT_TRUE( samurai::cellExists( mesh, Mesh_t::mesh_id_t::cells, static_cast(1), ij ) ); + } + + { + xt::xtensor_fixed> ij = {{0, 1}}; + ASSERT_FALSE( samurai::cellExists( mesh, Mesh_t::mesh_id_t::cells, static_cast(0), ij ) ); + } + + { + xt::xtensor_fixed> ij = {{0, 2}}; + ASSERT_TRUE( samurai::cellExists( mesh, Mesh_t::mesh_id_t::reference, static_cast(0), ij ) ); + } + + auto coords = make_field("coordinates", mesh); + auto level_field = make_field("level", mesh); + for_each_cell(mesh[Mesh_t::mesh_id_t::reference], [&](auto& cell) { + if constexpr ( dim == 1 ) + { + coords[cell] = cell.indices[0]; + } + else + { + coords[cell] = cell.indices; + } + level_field[cell] = cell.level; + }); + + samurai::save( "./", "test-lb-cellExists", {true, true}, mesh, coords, level_field ); + + } + #endif } From ca03f842b05275353d434630142ffd3090e99ed6 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 13 Jun 2024 16:19:20 +0200 Subject: [PATCH 143/170] fix compile error --- demos/FiniteVolume/advection_2d.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index e214301f9..b23b73838 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include From af040e1ad2a1357b4f1ed7893bf8e7d37e760e7d Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 13 Jun 2024 16:19:50 +0200 Subject: [PATCH 144/170] add new load balancing strategy squelette --- include/samurai/load_balancing_life.hpp | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 include/samurai/load_balancing_life.hpp diff --git a/include/samurai/load_balancing_life.hpp b/include/samurai/load_balancing_life.hpp new file mode 100644 index 000000000..d06ad0c38 --- /dev/null +++ b/include/samurai/load_balancing_life.hpp @@ -0,0 +1,83 @@ +#pragma once + +#include +#include "load_balancing.hpp" +#include + +// for std::sort +#include + +namespace Load_balancing{ + + class Life : public samurai::LoadBalancer { + + private: + + int _ndomains; + int _rank; + + public: + + Life() { + + #ifdef SAMURAI_WITH_MPI + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); + #else + _ndomains = 1; + _rank = 0; + #endif + } + + inline std::string getName() const { return "life"; } + + template + auto reordering_impl( Mesh_t & mesh ) { + auto flags = samurai::make_field("ordering_flag", mesh); + flags.fill( _rank ); + + return flags; + } + + template + auto load_balance_impl( Mesh_t & mesh ){ + + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + using CellList_t = typename Mesh_t::cl_type; + using mesh_id_t = typename Mesh_t::mesh_id_t; + + using Coord_t = xt::xtensor_fixed>; + using Stencil = xt::xtensor_fixed>; + + boost::mpi::communicator world; + + // For debug + // std::ofstream logs; + // logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); + logs << fmt::format("> New load-balancing using {} ", getName() ) << std::endl; + + auto flags = samurai::make_field("balancing_flags", mesh); + flags.fill( _rank ); + + // neighbourhood + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + + // fluxes to each neighbour + std::vector fluxes = samurai::cmptFluxes( mesh, 5 ); + + // cpy that can be modified + std::vector new_fluxes( fluxes ); + + // Loads of each processes + std::vector loads; + int my_load = static_cast( samurai::cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load, loads ); + + // samurai::cellExists(); + + return flags; + } + + }; +} \ No newline at end of file From 93a8c4c658d25a1a115cfd1b1e6667e0962a44b2 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 14 Jun 2024 13:46:40 +0200 Subject: [PATCH 145/170] fix MPI lock with update_mesh --- include/samurai/load_balancing.hpp | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index db939fe31..095609f72 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -573,9 +573,9 @@ namespace samurai [&](const auto& i, const auto& index) { std::copy(to_recv.begin() + count, - to_recv.begin() + count + static_cast(i.size()), + to_recv.begin() + count + static_cast(i.size()*field.size), new_field(level, i, index).begin()); - count += static_cast(i.size()); + count += static_cast(i.size()*field.size); // std::cerr << fmt::format("Process {}, recv interval {}", world.rank(), i) << std::endl; }); @@ -658,6 +658,7 @@ namespace samurai CellList_t new_cl; std::vector payload( static_cast( world.size() ) ); + std::vector payload_size( static_cast( world.size()), 0 ); std::map comm; @@ -678,13 +679,15 @@ namespace samurai if constexpr ( Mesh_t::dim == 1 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ {} ].add_point( cell.indices[ 0 ] ); } if constexpr ( Mesh_t::dim == 2 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ] } ].add_point( cell.indices[ 0 ] ); } if constexpr ( Mesh_t::dim == 3 ){ payload[ static_cast( flags[ cell ] ) ][ cell.level ][ { cell.indices[ 1 ], cell.indices[ 2 ] } ].add_point( cell.indices[ 0 ] ); } + + payload_size[ static_cast( flags[ cell ] ) ] ++; } }); logs << "\t\t>[Load_balancer::update_mesh] Comm required with processes : ["; for( const auto & it : comm ) - logs << it.first << ","; + logs << it.first << fmt::format(" ({} cells),", payload_size[it.first]); logs << "]" << std::endl; std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); @@ -696,21 +699,25 @@ namespace samurai int reqExchg; comm.find( iproc ) != comm.end() ? reqExchg = 1 : reqExchg = 0; + if( payload[ static_cast( iproc ) ].empty() ) reqExchg = 0; + req_send[ static_cast( iproc ) ] = reqExchg; - world.send( iproc, 17, reqExchg ); + world.send( iproc, 42, reqExchg ); } for( int iproc=0; iproc( iproc ) ] ); + world.recv( iproc, 42, req_recv[ static_cast( iproc ) ] ); } for( int iproc=0; iproc( iproc ) ], req_recv[ static_cast( iproc ) ] ) << std::endl;; } + std::vector req; + // actual data echange between processes that need to exchange data for(int iproc=0; iproc( iproc ) ] == 1 ){ CellArray_t to_send = { payload[ static_cast( iproc ) ], false }; - world.send( iproc, 17, to_send ); + req.push_back( world.isend( iproc, 17, to_send ) ); logs << fmt::format("\t> Sending to # {}", iproc ) << std::endl; } + } + + for(int iproc=0; iproc( iproc ) ] == 1 ) { CellArray_t to_rcv; @@ -734,6 +745,8 @@ namespace samurai } } + boost::mpi::wait_all( req.begin(), req.end() ); + Mesh_t new_mesh( new_cl, mesh ); return new_mesh; From f9af4803eb4d29a9c054f59edff33572e187937a Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 14 Jun 2024 13:46:52 +0200 Subject: [PATCH 146/170] add empty() to cellList --- include/samurai/cell_list.hpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/samurai/cell_list.hpp b/include/samurai/cell_list.hpp index 24a5da7ef..7cb128d54 100644 --- a/include/samurai/cell_list.hpp +++ b/include/samurai/cell_list.hpp @@ -34,6 +34,17 @@ namespace samurai const lcl_type& operator[](std::size_t i) const; lcl_type& operator[](std::size_t i); + inline bool empty() const { + + bool rst = true; + + for(const auto & lcl : m_cells ){ + rst &= lcl.empty(); + } + + return rst; + } + void to_stream(std::ostream& os) const; auto& origin_point() const; From ced6f23be5a6fd805da0e803713c2d907ebf3891 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 14 Jun 2024 13:47:12 +0200 Subject: [PATCH 147/170] update LBM demo --- demos/LBM/D2Q4444_Euler_Lax_Liu.cpp | 885 +++++++--------------------- 1 file changed, 207 insertions(+), 678 deletions(-) diff --git a/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp b/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp index 5cb94fcef..fd68fac25 100644 --- a/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp +++ b/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp @@ -4,44 +4,28 @@ #include #include -#include +#include +#include #include #include #include -#include +#include +#include #include -#include -#include "boundary_conditions.hpp" -#include "prediction_map_2d.hpp" +#include +#include -#include "utils_lbm_mr_2d.hpp" - -/// Timer used in tic & toc -auto tic_timer = std::chrono::high_resolution_clock::now(); - -/// Launching the timer -void tic() -{ - tic_timer = std::chrono::high_resolution_clock::now(); -} - -/// Stopping the timer and returning the duration in seconds -double toc() -{ - const auto toc_timer = std::chrono::high_resolution_clock::now(); - const std::chrono::duration time_span = toc_timer - tic_timer; - return time_span.count(); -} +#include double gm = 1.4; // Gas constant template -auto init_f(samurai::MROMesh& mesh, int config, double lambda) +auto init_f(samurai::MRMesh& mesh, int config, double lambda) { constexpr std::size_t nvel = 16; - using mesh_id_t = typename samurai::MROMesh::mesh_id_t; + using mesh_id_t = typename samurai::MRMesh::mesh_id_t; auto f = samurai::make_field("f", mesh); f.fill(0); @@ -192,79 +176,8 @@ auto init_f(samurai::MROMesh& mesh, int config, double lambda) return f; } -template -auto compute_prediction(std::size_t min_level, std::size_t max_level) -{ - coord_index_t i = 0, j = 0; - std::vector>> data(max_level - min_level + 1); - - auto rotation_of_pi_over_two = [](int alpha, int k, int h) - { - // Returns the rotation of (k, h) of an angle alpha * pi / 2. - // All the operations are performed on integer, to be exact - int cosinus = static_cast(std::round(std::cos(alpha * M_PI / 2.))); - int sinus = static_cast(std::round(std::sin(alpha * M_PI / 2.))); - - return std::pair(cosinus * k - sinus * h, sinus * k + cosinus * h); - }; - - // Transforms the coordinates to apply the rotation - auto tau = [](int delta, int k) - { - // The case in which delta = 0 is rather exceptional - if (delta == 0) - { - return k; - } - else - { - auto tmp = (1 << (delta - 1)); - return static_cast((k < tmp) ? (k - tmp) : (k - tmp + 1)); - } - }; - - auto tau_inverse = [](int delta, int k) - { - if (delta == 0) - { - return k; - } - else - { - auto tmp = (1 << (delta - 1)); - return static_cast((k < 0) ? (k + tmp) : (k + tmp - 1)); - } - }; - - for (std::size_t k = 0; k < max_level - min_level + 1; ++k) - { - int size = (1 << k); - data[k].resize(4); - - for (int alpha = 0; alpha < 4; ++alpha) - { - for (int l = 0; l < size; ++l) - { - // The reference direction from which the other ones are - // computed is that of (1, 0) - auto rotated_in = rotation_of_pi_over_two(alpha, tau(k, i * size - 1), tau(k, j * size + l)); - auto rotated_out = rotation_of_pi_over_two(alpha, tau(k, (i + 1) * size - 1), tau(k, j * size + l)); - - // For the cells inside the domain, we can already combine - // entering and exiting fluxes and we have a compensation of - // many cells. - data[k][alpha] += (prediction(k, tau_inverse(k, rotated_in.first), tau_inverse(k, rotated_in.second)) - - prediction(k, tau_inverse(k, rotated_out.first), tau_inverse(k, rotated_out.second))); - } - } - } - return data; -} - -template +template void one_time_step(Field& f, - Func&& update_bc_for_level, - const pred& pred_coeff, const double lambda, const double sq_rho, const double sxy_rho, @@ -273,193 +186,147 @@ void one_time_step(Field& f, const double sq_e, const double sxy_e) { - constexpr std::size_t nvel = Field::size; - using coord_index_t = typename Field::interval_t::coord_index_t; - - auto mesh = f.mesh(); - using mesh_id_t = typename decltype(mesh)::mesh_id_t; + constexpr std::size_t dim = Field::dim; + auto& mesh = f.mesh(); + using mesh_id_t = typename Field::mesh_t::mesh_id_t; - auto min_level = mesh.min_level(); - auto max_level = mesh.max_level(); - - samurai::update_ghost_mr(f, std::forward(update_bc_for_level)); - samurai::update_overleaves_mr(f, std::forward(update_bc_for_level)); + std::size_t max_level = mesh.max_level(); + samurai::update_ghost_mr(f); Field new_f{"new_f", mesh}; new_f.array().fill(0.); - Field fluxes{"fluxes", mesh}; // This stored the fluxes computed at the level of the overleaves - fluxes.array().fill(0.); Field advected{"advected", mesh}; advected.array().fill(0.); - for (std::size_t level = min_level; level <= max_level; ++level) - { - auto leaves = samurai::intersection(mesh[mesh_id_t::cells][level], mesh[mesh_id_t::cells][level]); - - if (level == max_level) - { // Advection at the finest level - - leaves( - [&](auto& interval, auto& index) - { - auto k = interval; // Logical index in x - auto h = index[0]; // Logical index in y - - // We enforce a bounce-back - for (int scheme_n = 0; scheme_n < 4; ++scheme_n) - { // We have 4 schemes - advected(0 + 4 * scheme_n, level, k, h) = f(0 + 4 * scheme_n, level, k - 1, h); - advected(1 + 4 * scheme_n, level, k, h) = f(1 + 4 * scheme_n, level, k, h - 1); - advected(2 + 4 * scheme_n, level, k, h) = f(2 + 4 * scheme_n, level, k + 1, h); - advected(3 + 4 * scheme_n, level, k, h) = f(3 + 4 * scheme_n, level, k, h + 1); - } - }); - } - else // Advection at the coarse levels using the overleaves + samurai::for_each_interval( + mesh[mesh_id_t::cells], + [&](std::size_t level, auto& i, auto& index) { - auto lev_p_1 = level + 1; - std::size_t j = max_level - (lev_p_1); - double coeff = 1. / (1 << (2 * j)); // ATTENTION A LA DIMENSION 2 !!!! - - leaves.on(level + 1)([&](auto& interval, auto& index) { // This are overleaves - auto k = interval; // Logical index in x - auto h = index[0]; // Logical index in y - - for (int scheme_n = 0; scheme_n < 4; ++scheme_n) - { - auto shift = 4 * scheme_n; - - for (std::size_t alpha = 0; alpha < 4; ++alpha) - { - for (auto& c : pred_coeff[j][alpha].coeff) - { - coord_index_t stencil_x, stencil_y; - std::tie(stencil_x, stencil_y) = c.first; - - fluxes(alpha + shift, lev_p_1, k, h) += c.second * f(alpha + shift, lev_p_1, k + stencil_x, h + stencil_y); - } - } - } - }); + auto j = index[0]; + auto jump = max_level - level; + double coef = 1. / (1 << (dim * jump)); + for (std::size_t scheme_n = 0; scheme_n < 4; ++scheme_n) + { // We have 4 schemes + advected(0 + 4 * scheme_n, level, i, j) = f(0 + 4 * scheme_n, level, i-1, j); + advected(1 + 4 * scheme_n, level, i, j) = f(1 + 4 * scheme_n, level, i, j-1); + advected(2 + 4 * scheme_n, level, i, j) = f(2 + 4 * scheme_n, level, i+1, j); + advected(3 + 4 * scheme_n, level, i, j) = f(3 + 4 * scheme_n, level, i, j+1); + + // advected( + // 0 + 4 * scheme_n, + // level, + // i, + // j) = f(0 + 4 * scheme_n, level, i, j) + // + coef * samurai::portion(f, 0 + 4 * scheme_n, level, i - 1, j, jump, {(1 << jump) - 1, (1 << jump)}, {0, (1 << jump)}) + // - coef * samurai::portion(f, 0 + 4 * scheme_n, level, i, j, jump, {(1 << jump) - 1, (1 << jump)}, {0, (1 << jump)}); + + // advected( + // 1 + 4 * scheme_n, + // level, + // i, + // j) = f(1 + 4 * scheme_n, level, i, j) + // + coef * samurai::portion(f, 1 + 4 * scheme_n, level, i, j - 1, jump, {0, (1 << jump)}, {(1 << jump) - 1, (1 << jump)}) + // - coef * samurai::portion(f, 1 + 4 * scheme_n, level, i, j, jump, {0, (1 << jump)}, {(1 << jump) - 1, (1 << jump)}); + + // advected(2 + 4 * scheme_n, level, i, j) = f( + // 2 + 4 * scheme_n, + // level, + // i, + // j) = f(2 + 4 * scheme_n, level, i, j) + // + coef * samurai::portion(f, 2 + 4 * scheme_n, level, i + 1, j, jump, {0, 1}, {0, (1 << jump)}) + // - coef * samurai::portion(f, 2 + 4 * scheme_n, level, i, j, jump, {0, 1}, {0, (1 << jump)}); + // advected(3 + 4 * scheme_n, + // level, + // i, + // j) = f(3 + 4 * scheme_n, level, i, j) + // + coef * samurai::portion(f, 3 + 4 * scheme_n, level, i, j + 1, jump, {0, (1 << jump)}, {0, 1}) + // - coef * samurai::portion(f, 3 + 4 * scheme_n, level, i, j, jump, {0, (1 << jump)}, {0, 1}); + } - leaves( - [&](auto& interval, auto& index) - { - auto k = interval; // Logical index in x - auto h = index[0]; // Logical index in y - - for (int alpha = 0; alpha < 16; ++alpha) - { - advected(alpha, level, k, h) = f(alpha, level, k, h) - + coeff * 0.25 - * (fluxes(alpha, lev_p_1, 2 * k, 2 * h) + fluxes(alpha, lev_p_1, 2 * k + 1, 2 * h) - + fluxes(alpha, lev_p_1, 2 * k, 2 * h + 1) - + fluxes(alpha, lev_p_1, 2 * k + 1, 2 * h + 1)); - } - }); - } + // We compute the advected momenti + auto m0_0 = xt::eval(advected(0, level, i, j) + advected(1, level, i, j) + advected(2, level, i, j) + advected(3, level, i, j)); + auto m0_1 = xt::eval(lambda * (advected(0, level, i, j) - advected(2, level, i, j))); + auto m0_2 = xt::eval(lambda * (advected(1, level, i, j) - advected(3, level, i, j))); + auto m0_3 = xt::eval( + lambda * lambda * (advected(0, level, i, j) - advected(1, level, i, j) + advected(2, level, i, j) - advected(3, level, i, j))); + + auto m1_0 = xt::eval(advected(4, level, i, j) + advected(5, level, i, j) + advected(6, level, i, j) + advected(7, level, i, j)); + auto m1_1 = xt::eval(lambda * (advected(4, level, i, j) - advected(6, level, i, j))); + auto m1_2 = xt::eval(lambda * (advected(5, level, i, j) - advected(7, level, i, j))); + auto m1_3 = xt::eval( + lambda * lambda * (advected(4, level, i, j) - advected(5, level, i, j) + advected(6, level, i, j) - advected(7, level, i, j))); + + auto m2_0 = xt::eval(advected(8, level, i, j) + advected(9, level, i, j) + advected(10, level, i, j) + advected(11, level, i, j)); + auto m2_1 = xt::eval(lambda * (advected(8, level, i, j) - advected(10, level, i, j))); + auto m2_2 = xt::eval(lambda * (advected(9, level, i, j) - advected(11, level, i, j))); + auto m2_3 = xt::eval( + lambda * lambda + * (advected(8, level, i, j) - advected(9, level, i, j) + advected(10, level, i, j) - advected(11, level, i, j))); + + auto m3_0 = xt::eval(advected(12, level, i, j) + advected(13, level, i, j) + advected(14, level, i, j) + + advected(15, level, i, j)); + auto m3_1 = xt::eval(lambda * (advected(12, level, i, j) - advected(14, level, i, j))); + auto m3_2 = xt::eval(lambda * (advected(13, level, i, j) - advected(15, level, i, j))); + auto m3_3 = xt::eval( + lambda * lambda + * (advected(12, level, i, j) - advected(13, level, i, j) + advected(14, level, i, j) - advected(15, level, i, j))); + + m0_1 = (1 - sq_rho) * m0_1 + sq_rho * (m1_0); + m0_2 = (1 - sq_rho) * m0_2 + sq_rho * (m2_0); + m0_3 = (1 - sxy_rho) * m0_3; + + m1_1 = (1 - sq_q) * m1_1 + + sq_q * ((3. / 2. - gm / 2.) * (m1_0 * m1_0) / (m0_0) + (1. / 2. - gm / 2.) * (m2_0 * m2_0) / (m0_0) + (gm - 1.) * m3_0); + m1_2 = (1 - sq_q) * m1_2 + sq_q * (m1_0 * m2_0 / m0_0); + m1_3 = (1 - sxy_q) * m1_3; + + m2_1 = (1 - sq_q) * m2_1 + sq_q * (m1_0 * m2_0 / m0_0); + m2_2 = (1 - sq_q) * m2_2 + + sq_q * ((3. / 2. - gm / 2.) * (m2_0 * m2_0) / (m0_0) + (1. / 2. - gm / 2.) * (m1_0 * m1_0) / (m0_0) + (gm - 1.) * m3_0); + m2_3 = (1 - sxy_q) * m2_3; + + m3_1 = (1 - sq_e) * m3_1 + + sq_e + * (gm * (m1_0 * m3_0) / (m0_0) - (gm / 2. - 1. / 2.) * (m1_0 * m1_0 * m1_0) / (m0_0 * m0_0) + - (gm / 2. - 1. / 2.) * (m1_0 * m2_0 * m2_0) / (m0_0 * m0_0)); + m3_2 = (1 - sq_e) * m3_2 + + sq_e + * (gm * (m2_0 * m3_0) / (m0_0) - (gm / 2. - 1. / 2.) * (m2_0 * m2_0 * m2_0) / (m0_0 * m0_0) + - (gm / 2. - 1. / 2.) * (m2_0 * m1_0 * m1_0) / (m0_0 * m0_0)); + m3_3 = (1 - sxy_e) * m3_3; + + new_f(0, level, i, j) = .25 * m0_0 + .5 / lambda * (m0_1) + .25 / (lambda * lambda) * m0_3; + new_f(1, level, i, j) = .25 * m0_0 + .5 / lambda * (m0_2)-.25 / (lambda * lambda) * m0_3; + new_f(2, level, i, j) = .25 * m0_0 - .5 / lambda * (m0_1) + .25 / (lambda * lambda) * m0_3; + new_f(3, level, i, j) = .25 * m0_0 - .5 / lambda * (m0_2)-.25 / (lambda * lambda) * m0_3; + + new_f(4, level, i, j) = .25 * m1_0 + .5 / lambda * (m1_1) + .25 / (lambda * lambda) * m1_3; + new_f(5, level, i, j) = .25 * m1_0 + .5 / lambda * (m1_2)-.25 / (lambda * lambda) * m1_3; + new_f(6, level, i, j) = .25 * m1_0 - .5 / lambda * (m1_1) + .25 / (lambda * lambda) * m1_3; + new_f(7, level, i, j) = .25 * m1_0 - .5 / lambda * (m1_2)-.25 / (lambda * lambda) * m1_3; + + new_f(8, level, i, j) = .25 * m2_0 + .5 / lambda * (m2_1) + .25 / (lambda * lambda) * m2_3; + new_f(9, level, i, j) = .25 * m2_0 + .5 / lambda * (m2_2)-.25 / (lambda * lambda) * m2_3; + new_f(10, level, i, j) = .25 * m2_0 - .5 / lambda * (m2_1) + .25 / (lambda * lambda) * m2_3; + new_f(11, level, i, j) = .25 * m2_0 - .5 / lambda * (m2_2)-.25 / (lambda * lambda) * m2_3; + + new_f(12, level, i, j) = .25 * m3_0 + .5 / lambda * (m3_1) + .25 / (lambda * lambda) * m3_3; + new_f(13, level, i, j) = .25 * m3_0 + .5 / lambda * (m3_2)-.25 / (lambda * lambda) * m3_3; + new_f(14, level, i, j) = .25 * m3_0 - .5 / lambda * (m3_1) + .25 / (lambda * lambda) * m3_3; + new_f(15, level, i, j) = .25 * m3_0 - .5 / lambda * (m3_2)-.25 / (lambda * lambda) * m3_3; + }); - leaves( - [&](auto& interval, auto& index) - { - auto k = interval; // Logical index in x - auto h = index[0]; // Logical index in y - - // We compute the advected momenti - auto m0_0 = xt::eval(advected(0, level, k, h) + advected(1, level, k, h) + advected(2, level, k, h) - + advected(3, level, k, h)); - auto m0_1 = xt::eval(lambda * (advected(0, level, k, h) - advected(2, level, k, h))); - auto m0_2 = xt::eval(lambda * (advected(1, level, k, h) - advected(3, level, k, h))); - auto m0_3 = xt::eval( - lambda * lambda - * (advected(0, level, k, h) - advected(1, level, k, h) + advected(2, level, k, h) - advected(3, level, k, h))); - - auto m1_0 = xt::eval(advected(4, level, k, h) + advected(5, level, k, h) + advected(6, level, k, h) - + advected(7, level, k, h)); - auto m1_1 = xt::eval(lambda * (advected(4, level, k, h) - advected(6, level, k, h))); - auto m1_2 = xt::eval(lambda * (advected(5, level, k, h) - advected(7, level, k, h))); - auto m1_3 = xt::eval( - lambda * lambda - * (advected(4, level, k, h) - advected(5, level, k, h) + advected(6, level, k, h) - advected(7, level, k, h))); - - auto m2_0 = xt::eval(advected(8, level, k, h) + advected(9, level, k, h) + advected(10, level, k, h) - + advected(11, level, k, h)); - auto m2_1 = xt::eval(lambda * (advected(8, level, k, h) - advected(10, level, k, h))); - auto m2_2 = xt::eval(lambda * (advected(9, level, k, h) - advected(11, level, k, h))); - auto m2_3 = xt::eval( - lambda * lambda - * (advected(8, level, k, h) - advected(9, level, k, h) + advected(10, level, k, h) - advected(11, level, k, h))); - - auto m3_0 = xt::eval(advected(12, level, k, h) + advected(13, level, k, h) + advected(14, level, k, h) - + advected(15, level, k, h)); - auto m3_1 = xt::eval(lambda * (advected(12, level, k, h) - advected(14, level, k, h))); - auto m3_2 = xt::eval(lambda * (advected(13, level, k, h) - advected(15, level, k, h))); - auto m3_3 = xt::eval( - lambda * lambda - * (advected(12, level, k, h) - advected(13, level, k, h) + advected(14, level, k, h) - advected(15, level, k, h))); - - m0_1 = (1 - sq_rho) * m0_1 + sq_rho * (m1_0); - m0_2 = (1 - sq_rho) * m0_2 + sq_rho * (m2_0); - m0_3 = (1 - sxy_rho) * m0_3; - - m1_1 = (1 - sq_q) * m1_1 - + sq_q * ((3. / 2. - gm / 2.) * (m1_0 * m1_0) / (m0_0) + (1. / 2. - gm / 2.) * (m2_0 * m2_0) / (m0_0) + (gm - 1.) * m3_0); - m1_2 = (1 - sq_q) * m1_2 + sq_q * (m1_0 * m2_0 / m0_0); - m1_3 = (1 - sxy_q) * m1_3; - - m2_1 = (1 - sq_q) * m2_1 + sq_q * (m1_0 * m2_0 / m0_0); - m2_2 = (1 - sq_q) * m2_2 - + sq_q * ((3. / 2. - gm / 2.) * (m2_0 * m2_0) / (m0_0) + (1. / 2. - gm / 2.) * (m1_0 * m1_0) / (m0_0) + (gm - 1.) * m3_0); - m2_3 = (1 - sxy_q) * m2_3; - - m3_1 = (1 - sq_e) * m3_1 - + sq_e - * (gm * (m1_0 * m3_0) / (m0_0) - (gm / 2. - 1. / 2.) * (m1_0 * m1_0 * m1_0) / (m0_0 * m0_0) - - (gm / 2. - 1. / 2.) * (m1_0 * m2_0 * m2_0) / (m0_0 * m0_0)); - m3_2 = (1 - sq_e) * m3_2 - + sq_e - * (gm * (m2_0 * m3_0) / (m0_0) - (gm / 2. - 1. / 2.) * (m2_0 * m2_0 * m2_0) / (m0_0 * m0_0) - - (gm / 2. - 1. / 2.) * (m2_0 * m1_0 * m1_0) / (m0_0 * m0_0)); - m3_3 = (1 - sxy_e) * m3_3; - - new_f(0, level, k, h) = .25 * m0_0 + .5 / lambda * (m0_1) + .25 / (lambda * lambda) * m0_3; - new_f(1, level, k, h) = .25 * m0_0 + .5 / lambda * (m0_2)-.25 / (lambda * lambda) * m0_3; - new_f(2, level, k, h) = .25 * m0_0 - .5 / lambda * (m0_1) + .25 / (lambda * lambda) * m0_3; - new_f(3, level, k, h) = .25 * m0_0 - .5 / lambda * (m0_2)-.25 / (lambda * lambda) * m0_3; - - new_f(4, level, k, h) = .25 * m1_0 + .5 / lambda * (m1_1) + .25 / (lambda * lambda) * m1_3; - new_f(5, level, k, h) = .25 * m1_0 + .5 / lambda * (m1_2)-.25 / (lambda * lambda) * m1_3; - new_f(6, level, k, h) = .25 * m1_0 - .5 / lambda * (m1_1) + .25 / (lambda * lambda) * m1_3; - new_f(7, level, k, h) = .25 * m1_0 - .5 / lambda * (m1_2)-.25 / (lambda * lambda) * m1_3; - - new_f(8, level, k, h) = .25 * m2_0 + .5 / lambda * (m2_1) + .25 / (lambda * lambda) * m2_3; - new_f(9, level, k, h) = .25 * m2_0 + .5 / lambda * (m2_2)-.25 / (lambda * lambda) * m2_3; - new_f(10, level, k, h) = .25 * m2_0 - .5 / lambda * (m2_1) + .25 / (lambda * lambda) * m2_3; - new_f(11, level, k, h) = .25 * m2_0 - .5 / lambda * (m2_2)-.25 / (lambda * lambda) * m2_3; - - new_f(12, level, k, h) = .25 * m3_0 + .5 / lambda * (m3_1) + .25 / (lambda * lambda) * m3_3; - new_f(13, level, k, h) = .25 * m3_0 + .5 / lambda * (m3_2)-.25 / (lambda * lambda) * m3_3; - new_f(14, level, k, h) = .25 * m3_0 - .5 / lambda * (m3_1) + .25 / (lambda * lambda) * m3_3; - new_f(15, level, k, h) = .25 * m3_0 - .5 / lambda * (m3_2)-.25 / (lambda * lambda) * m3_3; - }); - } std::swap(f.array(), new_f.array()); } template -void save_solution(Field& f, double eps, std::size_t ite, std::string ext = "") +void save_solution(Field& f, double eps, std::size_t ite, std::size_t freq_out, std::string ext = "") { using value_t = typename Field::value_type; - auto mesh = f.mesh(); - using mesh_id_t = typename decltype(mesh)::mesh_id_t; - - std::size_t min_level = mesh.min_level(); - std::size_t max_level = mesh.max_level(); - - std::stringstream str; - str << "LBM_D2Q4_3_Euler_" << ext << "_lmin_" << min_level << "_lmax-" << max_level << "_eps-" << eps << "_ite-" << ite; + if( ite % freq_out != 0 ) return; + auto& mesh = f.mesh(); auto level = samurai::make_field("level", mesh); auto rho = samurai::make_field("rho", mesh); auto qx = samurai::make_field("qx", mesh); @@ -467,7 +334,7 @@ void save_solution(Field& f, double eps, std::size_t ite, std::string ext = "") auto e = samurai::make_field("e", mesh); auto s = samurai::make_field("entropy", mesh); - samurai::for_each_cell(mesh[mesh_id_t::cells], + samurai::for_each_cell(mesh, [&](auto& cell) { level[cell] = cell.level; @@ -482,449 +349,111 @@ void save_solution(Field& f, double eps, std::size_t ite, std::string ext = "") s[cell] = std::log(p / std::pow(rho[cell], gm)); }); - samurai::save(str.str().data(), mesh, rho, qx, qy, e, s, f, level); + std::size_t min_level = mesh.min_level(); + std::size_t max_level = mesh.max_level(); + std::string filename = fmt::format("LBM_D2Q4_3_Euler_{}_lmin-{}_lmax-{}_eps-{}_ite-{}", ext, min_level, max_level, eps, ite); + samurai::save(filename, mesh, rho, qx, qy, e, s, f, level); } -// Attention : the number 2 as second template parameter does not mean -// that we are dealing with two fields!!!! -template -xt::xtensor -prediction_all(const Field& f, - std::size_t level_g, - std::size_t level, - const interval_t& k, - const ordinates_t& h, - std::map, xt::xtensor>& mem_map) +int main(int argc, char* argv[]) { - // That is used to employ _ with xtensor - using namespace xt::placeholders; - - // mem_map.clear(); // To be activated if we want to avoid memoization - auto it = mem_map.find({level_g, level, k, h}); - - if (it != mem_map.end() && k.size() == (std::get<2>(it->first)).size()) - { - return it->second; - } - else - { - auto mesh = f.mesh(); - using mesh_id_t = typename decltype(mesh)::mesh_id_t; - - // We put only the size in x (k.size()) because in y we only have slices - // of size 1. The second term (1) should be adapted according to the - // number of fields that we have. - std::vector shape_x = {k.size(), 16}; - xt::xtensor out = xt::empty(shape_x); - auto mask = mesh.exists(mesh_id_t::cells_and_ghosts, - level_g + level, - k, - h); // Check if we are on a leaf or a ghost (CHECK IF IT IS OK) - xt::xtensor mask_all = xt::empty(shape_x); - - for (int h_field = 0; h_field < 16; ++h_field) - { - xt::view(mask_all, xt::all(), h_field) = mask; - } - - if (xt::all(mask)) // Recursion finished - { - return xt::eval(f(0, 16, level_g + level, k, h)); - } + samurai::initialize(argc, argv); - // If we cannot stop here - auto kg = k >> 1; - kg.step = 1; - xt::xtensor val = xt::empty(shape_x); - /* - -------------------- - NW | N | NE - -------------------- - W | EARTH | E - -------------------- - SW | S | SE - -------------------- - */ - - auto earth = xt::eval(prediction_all(f, level_g, level - 1, kg, (h >> 1), mem_map)); - auto W = xt::eval(prediction_all(f, level_g, level - 1, kg - 1, (h >> 1), mem_map)); - auto E = xt::eval(prediction_all(f, level_g, level - 1, kg + 1, (h >> 1), mem_map)); - auto S = xt::eval(prediction_all(f, level_g, level - 1, kg, (h >> 1) - 1, mem_map)); - auto N = xt::eval(prediction_all(f, level_g, level - 1, kg, (h >> 1) + 1, mem_map)); - auto SW = xt::eval(prediction_all(f, level_g, level - 1, kg - 1, (h >> 1) - 1, mem_map)); - auto SE = xt::eval(prediction_all(f, level_g, level - 1, kg + 1, (h >> 1) - 1, mem_map)); - auto NW = xt::eval(prediction_all(f, level_g, level - 1, kg - 1, (h >> 1) + 1, mem_map)); - auto NE = xt::eval(prediction_all(f, level_g, level - 1, kg + 1, (h >> 1) + 1, mem_map)); - - // This is to deal with odd/even indices in the x direction - std::size_t start_even = (k.start & 1) ? 1 : 0; - std::size_t start_odd = (k.start & 1) ? 0 : 1; - std::size_t end_even = (k.end & 1) ? kg.size() : kg.size() - 1; - std::size_t end_odd = (k.end & 1) ? kg.size() - 1 : kg.size(); - - int delta_y = (h & 1) ? 1 : 0; - int m1_delta_y = (delta_y == 0) ? 1 : -1; // (-1)^(delta_y) - - xt::view(val, xt::range(start_even, _, 2)) = xt::view( - earth + 1. / 8 * (W - E) + 1. / 8 * m1_delta_y * (S - N) - 1. / 64 * m1_delta_y * (NE - NW - SE + SW), - xt::range(start_even, _)); - xt::view(val, xt::range(start_odd, _, 2)) = xt::view( - earth - 1. / 8 * (W - E) + 1. / 8 * m1_delta_y * (S - N) + 1. / 64 * m1_delta_y * (NE - NW - SE + SW), - xt::range(_, end_odd)); - - xt::masked_view(out, !mask_all) = xt::masked_view(val, !mask_all); - - for (int k_mask = 0, k_int = k.start; k_int < k.end; ++k_mask, ++k_int) - { - if (mask[k_mask]) - { - xt::view(out, k_mask) = xt::view(f(0, 16, level_g + level, {k_int, k_int + 1}, h), 0); - } - } + CLI::App app{"Multi resolution for a D2Q4 LBM scheme for the scalar advection equation"}; - // It is crucial to use insert and not [] in order not to update the - // value in case of duplicated (same key) - mem_map.insert(std::make_pair(std::tuple{level_g, level, k, h}, out)); - return out; - } -} + std::size_t freq_out = 1; // frequency for output in iteration + std::size_t total_nb_ite = 100; + int configuration = 12; -template -double compute_error(Field& f, FieldFull& f_full, Func&& update_bc_for_level) -{ - constexpr std::size_t size = Field::size; - using value_t = typename Field::value_type; + // Multiresolution parameters + std::size_t min_level = 2; + std::size_t max_level = 9; + double mr_epsilon = 1.e-3; // Threshold used by multiresolution + double mr_regularity = 2.; // Regularity guess for multiresolution - auto mesh = f.mesh(); - using mesh_id_t = typename decltype(mesh)::mesh_id_t; + std::size_t nt_loadbalance = 10; - auto min_level = mesh.min_level(); - auto max_level = mesh.max_level(); + app.add_option("--nb-ite", total_nb_ite, "number of iteration")->capture_default_str(); + app.add_option("--config", configuration, "Lax-Liu configuration")->capture_default_str(); + app.add_option("--min-level", min_level, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--max-level", max_level, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--nt-loadbalance", nt_loadbalance, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-eps", mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh")->capture_default_str() + ->group("Multiresolution"); + app.add_option("--mr-reg", mr_regularity, "The regularity criteria used by the multiresolution to adapt the mesh") + ->capture_default_str()->group("Multiresolution"); + CLI11_PARSE(app, argc, argv); - auto init_mesh = f_full.mesh(); + constexpr size_t dim = 2; + using Config = samurai::MRConfig; - samurai::update_ghost_mr(f, std::forward(update_bc_for_level)); + double lambda = 1. / 0.2; // This seems to work + double T = 0.25; // 0.3;//1.2; - auto f_reconstructed = samurai::make_field("f_reconstructed", init_mesh); - f_reconstructed.fill(0.); + double sq_rho = 1.9; + double sxy_rho = 1.; - // For memoization - using interval_t = typename Field::interval_t; // Type in X - using ordinates_t = typename interval_t::index_t; // Type in Y - std::map, xt::xtensor> memoization_map; - memoization_map.clear(); + double sq_q = 1.75; + double sxy_q = 1.; - double error = 0.; - double norm = 0.; - double dx = 1. / (1 << max_level); + double sq_e = 1.75; + double sxy_e = 1.; - for (std::size_t level = 0; level <= max_level; ++level) + if (configuration == 12) { - auto leaves_on_finest = samurai::intersection(mesh[mesh_id_t::cells][level], mesh[mesh_id_t::cells][level]); - - leaves_on_finest.on(max_level)( - [&](auto& interval, auto& index) - { - auto k = interval; - auto h = index[0]; - - f_reconstructed(max_level, k, h) = prediction_all(f, level, max_level - level, k, h, memoization_map); - - auto rho_reconstructed = f_reconstructed(0, max_level, k, h) + f_reconstructed(1, max_level, k, h) - + f_reconstructed(2, max_level, k, h) + f_reconstructed(3, max_level, k, h); - auto rho_full = f_full(0, max_level, k, h) + f_full(1, max_level, k, h) + f_full(2, max_level, k, h) - + f_full(3, max_level, k, h); - - error += xt::sum(xt::abs(rho_reconstructed - rho_full))[0]; - norm += xt::sum(xt::abs(rho_full))[0]; - }); + T = .25; } - return (error / norm); -} - -template -void save_reconstructed(Field& f, FieldFull& f_full, Func&& update_bc_for_level, double eps, std::size_t ite, std::string ext = "") -{ - constexpr std::size_t size = Field::size; - using value_t = typename Field::value_type; - - auto mesh = f.mesh(); - using mesh_id_t = typename decltype(mesh)::mesh_id_t; - - auto min_level = mesh.min_level(); - auto max_level = mesh.max_level(); - - auto init_mesh = f_full.mesh(); - - samurai::update_ghost_mr(f, std::forward(update_bc_for_level)); - - auto f_reconstructed = samurai::make_field("f_reconstructed", init_mesh); // To reconstruct all and - // see entropy - f_reconstructed.fill(0.); - - auto rho_reconstructed = samurai::make_field("rho_reconstructed", init_mesh); - auto qx_reconstructed = samurai::make_field("qx_reconstructed", init_mesh); - auto qy_reconstructed = samurai::make_field("qy_reconstructed", init_mesh); - auto E_reconstructed = samurai::make_field("E_reconstructed", init_mesh); - auto s_reconstructed = samurai::make_field("s_reconstructed", init_mesh); - auto level_ = samurai::make_field("level", init_mesh); - - auto rho = samurai::make_field("rho", init_mesh); - auto qx = samurai::make_field("qx", init_mesh); - auto qy = samurai::make_field("qy", init_mesh); - auto E = samurai::make_field("E", init_mesh); - auto s = samurai::make_field("s", init_mesh); - - // For memoization - using interval_t = typename Field::interval_t; // Type in X - using ordinates_t = typename interval_t::index_t; // Type in Y - std::map, xt::xtensor> memoization_map; - - memoization_map.clear(); - - for (std::size_t level = 0; level <= max_level; ++level) + else { - auto number_leaves = mesh.nb_cells(level, mesh_id_t::cells); - - auto leaves_on_finest = samurai::intersection(mesh[mesh_id_t::cells][level], mesh[mesh_id_t::cells][level]); - - leaves_on_finest.on(max_level)( - [&](auto& interval, auto& index) - { - auto k = interval; - auto h = index[0]; - - f_reconstructed(max_level, k, h) = prediction_all(f, level, max_level - level, k, h, memoization_map); - - level_(max_level, k, h) = level; - - rho_reconstructed(max_level, k, h) = f_reconstructed(0, max_level, k, h) + f_reconstructed(1, max_level, k, h) - + f_reconstructed(2, max_level, k, h) + f_reconstructed(3, max_level, k, h); - - qx_reconstructed(max_level, k, h) = f_reconstructed(4, max_level, k, h) + f_reconstructed(5, max_level, k, h) - + f_reconstructed(6, max_level, k, h) + f_reconstructed(7, max_level, k, h); - - qy_reconstructed(max_level, k, h) = f_reconstructed(8, max_level, k, h) + f_reconstructed(9, max_level, k, h) - + f_reconstructed(10, max_level, k, h) + f_reconstructed(11, max_level, k, h); - - E_reconstructed(max_level, k, h) = f_reconstructed(12, max_level, k, h) + f_reconstructed(13, max_level, k, h) - + f_reconstructed(14, max_level, k, h) + f_reconstructed(15, max_level, k, h); - - s_reconstructed(max_level, k, h) = xt::log( - ((gm - 1.) - * (E_reconstructed(max_level, k, h) - - .5 * (xt::pow(qx_reconstructed(max_level, k, h), 2.) + xt::pow(qy_reconstructed(max_level, k, h), 2.)) - / rho_reconstructed(max_level, k, h))) - / xt::pow(rho_reconstructed(max_level, k, h), gm)); - - rho(max_level, k, h) = f_full(0, max_level, k, h) + f_full(1, max_level, k, h) + f_full(2, max_level, k, h) - + f_full(3, max_level, k, h); + T = .3; + // T = 0.1; + } - qx(max_level, k, h) = f_full(4, max_level, k, h) + f_full(5, max_level, k, h) + f_full(6, max_level, k, h) - + f_full(7, max_level, k, h); + samurai::Box box({0, 0}, {1, 1}); + samurai::MRMesh mesh(box, min_level, max_level); - qy(max_level, k, h) = f_full(8, max_level, k, h) + f_full(9, max_level, k, h) + f_full(10, max_level, k, h) - + f_full(11, max_level, k, h); + // Initialization + auto f = init_f(mesh, configuration, lambda); // Adaptive scheme + samurai::make_bc>(f, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.); - E(max_level, k, h) = f_full(12, max_level, k, h) + f_full(13, max_level, k, h) + f_full(14, max_level, k, h) - + f_full(15, max_level, k, h); + double dx = 1.0 / (1 << max_level); + double dt = dx / lambda; - s(max_level, k, h) = xt::log( - ((gm - 1.) - * (E(max_level, k, h) - .5 * (xt::pow(qx(max_level, k, h), 2.) + xt::pow(qy(max_level, k, h), 2.)) / rho(max_level, k, h))) - / xt::pow(rho(max_level, k, h), gm)); - }); - } + std::size_t N = static_cast(T / dt); - std::stringstream str; - str << "LBM_D2Q4_3_Euler_Reconstruction_" << ext << "_lmin_" << min_level << "_lmax-" << max_level << "_eps-" << eps << "_ite-" << ite; - - samurai::save(str.str().data(), - init_mesh, - rho_reconstructed, - qx_reconstructed, - qy_reconstructed, - E_reconstructed, - s_reconstructed, - rho, - qx, - qy, - E, - s, - level_); -} + SFC_LoadBalancer_interval balancer; -int main(int argc, char* argv[]) -{ - samurai::initialize(argc, argv); + auto MRadaptation = samurai::make_MRAdapt(f); - cxxopts::Options options("lbm_d2q4_3_Euler", - "Multi resolution for a D2Q4 LBM scheme for the " - "scalar advection equation"); - - options.add_options()("min_level", "minimum level", cxxopts::value()->default_value("2"))( - "max_level", - "maximum level", - cxxopts::value()->default_value("7"))("epsilon", "maximum level", cxxopts::value()->default_value("0.0001"))( - "ite", - "number of iteration", - cxxopts::value()->default_value("100"))("reg", "regularity", cxxopts::value()->default_value("0."))( - "config", - "Lax-Liu configuration", - cxxopts::value()->default_value("12"))("h, help", "Help"); - - try + double t = 0.; + samurai::times::timers.start("tloop"); + for (std::size_t nt = 0; nt <= N && nt < total_nb_ite; ++nt) { - auto result = options.parse(argc, argv); + std::cout << fmt::format("\n\t> Iteration {}, t: {}, dt: {} ", nt, t, dt ) << std::endl; - if (result.count("help")) + if (nt % nt_loadbalance == 0 && nt > 1) { - std::cout << options.help() << "\n"; + samurai::times::timers.start("tloop.lb"); + balancer.load_balance(mesh, f); + samurai::times::timers.stop("tloop.lb"); } - else - { - constexpr size_t dim = 2; - using Config = samurai::MROConfig; - - std::size_t min_level = result["min_level"].as(); - std::size_t max_level = result["max_level"].as(); - std::size_t total_nb_ite = result["ite"].as(); - double eps = result["epsilon"].as(); - double regularity = result["reg"].as(); - int configuration = result["config"].as(); - - // double lambda = 1./0.3; //4.0; - // double lambda = 1./0.2499; //4.0; - double lambda = 1. / 0.2; // This seems to work - double T = 0.25; // 0.3;//1.2; - double sq_rho = 1.9; - double sxy_rho = 1.; + samurai::times::timers.start("tloop.MRAdaptation"); + MRadaptation(mr_epsilon, mr_regularity); + samurai::times::timers.stop("tloop.MRAdaptation"); - double sq_q = 1.75; - double sxy_q = 1.; + samurai::times::timers.start("tloop.LBM"); + one_time_step(f, lambda, sq_rho, sxy_rho, sq_q, sxy_q, sq_e, sxy_e); + samurai::times::timers.stop("tloop.LBM"); - double sq_e = 1.75; - double sxy_e = 1.; + samurai::times::timers.start("tloop.io"); + save_solution(f, mr_epsilon, nt, freq_out); + samurai::times::timers.stop("tloop.io"); - if (configuration == 12) - { - T = .25; - } - else - { - T = .3; - // T = 0.1; - } - - // // This were the old test case (version 3) - // double sq = 1.75; - // double sxy = 2.; - // if (configuration == 12) { - // sxy = 1.5; - // T = 0.25; - // } - // else { - // sxy = 0.5; - // T = 0.3; - // } - - samurai::Box box({0, 0}, {1, 1}); - samurai::MROMesh mesh(box, min_level, max_level); - using mesh_id_t = typename samurai::MROMesh::mesh_id_t; - samurai::MROMesh mesh_ref{box, max_level, max_level}; - - using coord_index_t = typename samurai::MROMesh::coord_index_t; - auto pred_coeff = compute_prediction(min_level, max_level); - - // Initialization - auto f = init_f(mesh, configuration, lambda); // Adaptive scheme - auto f_ref = init_f(mesh_ref, configuration, lambda); // Reference scheme - - double dx = 1.0 / (1 << max_level); - double dt = dx / lambda; - - std::size_t N = static_cast(T / dt); - - std::string dirname("./LaxLiu/"); - std::string suffix("_Config_" + std::to_string(configuration) + "_min_" + std::to_string(min_level) + "_max_" - + std::to_string(max_level) + "_eps_" + std::to_string(eps)); - - // std::ofstream stream_number_leaves; - // stream_number_leaves.open - // (dirname+"number_leaves"+suffix+".dat"); - - // std::ofstream stream_number_cells; - // stream_number_cells.open (dirname+"number_cells"+suffix+".dat"); - - // std::ofstream stream_time_scheme_ref; - // stream_time_scheme_ref.open - // (dirname+"time_scheme_ref"+suffix+".dat"); - - // std::ofstream stream_number_leaves_ref; - // stream_number_leaves_ref.open - // (dirname+"number_leaves_ref"+suffix+".dat"); - - // std::ofstream stream_number_cells_ref; - // stream_number_cells_ref.open - // (dirname+"number_cells_ref"+suffix+".dat"); - - int howoften = 1; // How often is the solution saved ? - - auto update_bc_for_level = [](auto& field, std::size_t level) - { - update_bc_D2Q4_3_Euler_constant_extension(field, level); - }; - - auto MRadaptation = samurai::make_MRAdapt(f, update_bc_for_level); - - for (std::size_t nb_ite = 0; nb_ite <= N; ++nb_ite) - { - std::cout << std::endl << " Iteration number = " << nb_ite << std::endl; - - if (max_level > min_level) - { - MRadaptation(eps, regularity); - } - - if (nb_ite == N) - { - auto error_density = compute_error(f, f_ref, update_bc_for_level); - std::cout << std::endl << "#### Epsilon = " << eps << " error = " << error_density << std::endl; - save_solution(f, eps, nb_ite, std::string("final_")); - save_reconstructed(f, f_ref, update_bc_for_level, eps, nb_ite); - } - - // if (nb_ite % howoften == 0) { - // save_solution(f , eps, nb_ite/howoften, - // std::string("Config_")+std::to_string(configuration)); // - // Before applying the scheme - // } - - one_time_step(f, update_bc_for_level, pred_coeff, lambda, sq_rho, sxy_rho, sq_q, sxy_q, sq_e, sxy_e); - one_time_step(f_ref, update_bc_for_level, pred_coeff, lambda, sq_rho, sxy_rho, sq_q, sxy_q, sq_e, sxy_e); - - auto number_leaves = mesh.nb_cells(mesh_id_t::cells); - auto number_cells = mesh.nb_cells(); - - samurai::statistics("D2Q4444_Euler_Lax_Liu", mesh); - // stream_number_leaves< Date: Fri, 14 Jun 2024 13:49:37 +0200 Subject: [PATCH 148/170] fix missing link libraries for lbm --- demos/LBM/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/LBM/CMakeLists.txt b/demos/LBM/CMakeLists.txt index 71d959b9d..2eb421061 100644 --- a/demos/LBM/CMakeLists.txt +++ b/demos/LBM/CMakeLists.txt @@ -30,7 +30,7 @@ target_link_libraries(lbm-D1Q222-euler-sod samurai) # 2D Paper add_executable(lbm-D2Q4444-euler-lax-liu D2Q4444_Euler_Lax_Liu.cpp) -target_link_libraries(lbm-D2Q4444-euler-lax-liu samurai) +target_link_libraries(lbm-D2Q4444-euler-lax-liu samurai CLI11::CLI11 ${MPI_LIBRARIES}) add_executable(lbm-D2Q4444-euler-lax-liu-uniform D2Q4444_Euler_Lax_Liu_uniform.cpp) target_link_libraries(lbm-D2Q4444-euler-lax-liu-uniform samurai) From 77f58929b180800f081de6968407a3a9b8cb3c95 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 14 Jun 2024 13:49:51 +0200 Subject: [PATCH 149/170] uncomment lbm in cmake --- demos/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/CMakeLists.txt b/demos/CMakeLists.txt index 4dcafe466..c4dc0d812 100644 --- a/demos/CMakeLists.txt +++ b/demos/CMakeLists.txt @@ -5,7 +5,7 @@ add_subdirectory(from_obj) add_subdirectory(FiniteVolume) add_subdirectory(loadbalancing) # add_subdirectory(MPI) -# add_subdirectory(LBM) +add_subdirectory(LBM) add_subdirectory(p4est) add_subdirectory(pablo) add_subdirectory(tutorial) From c7bddb25b6acb59c9b922ff68787e694914244b8 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 14 Jun 2024 15:54:48 +0200 Subject: [PATCH 150/170] add more load balancing strategy in LBM demo --- demos/LBM/D2Q4444_Euler_Lax_Liu.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp b/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp index fd68fac25..f412036a6 100644 --- a/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp +++ b/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp @@ -16,7 +16,11 @@ #include #include - +#include +#include +#include +#include +#include #include double gm = 1.4; // Gas constant @@ -421,7 +425,15 @@ int main(int argc, char* argv[]) std::size_t N = static_cast(T / dt); - SFC_LoadBalancer_interval balancer; + // SFC_LoadBalancer_interval balancer; + // SFC_LoadBalancer_interval balancer; + // Load_balancing::Life balancer; + // Void_LoadBalancer balancer; + Diffusion_LoadBalancer_cell balancer; + // Diffusion_LoadBalancer_interval balancer; + // Load_balancing::Diffusion balancer; + + auto MRadaptation = samurai::make_MRAdapt(f); From 42a0eebf690a8ea615b6c62bfcd498c633b1e17c Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 14 Jun 2024 16:40:59 +0200 Subject: [PATCH 151/170] update timer in lbm demo --- demos/LBM/D2Q4444_Euler_Lax_Liu.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp b/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp index f412036a6..8ec94cd52 100644 --- a/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp +++ b/demos/LBM/D2Q4444_Euler_Lax_Liu.cpp @@ -195,13 +195,19 @@ void one_time_step(Field& f, using mesh_id_t = typename Field::mesh_t::mesh_id_t; std::size_t max_level = mesh.max_level(); + + samurai::times::timers.start("ugm-step"); samurai::update_ghost_mr(f); + samurai::times::timers.stop("ugm-step"); + samurai::times::timers.start("field-step"); Field new_f{"new_f", mesh}; new_f.array().fill(0.); Field advected{"advected", mesh}; advected.array().fill(0.); + samurai::times::timers.stop("field-step"); + samurai::times::timers.start("lbm-step"); samurai::for_each_interval( mesh[mesh_id_t::cells], [&](std::size_t level, auto& i, auto& index) @@ -319,6 +325,8 @@ void one_time_step(Field& f, new_f(14, level, i, j) = .25 * m3_0 - .5 / lambda * (m3_1) + .25 / (lambda * lambda) * m3_3; new_f(15, level, i, j) = .25 * m3_0 - .5 / lambda * (m3_2)-.25 / (lambda * lambda) * m3_3; }); + + samurai::times::timers.stop("lbm-step"); std::swap(f.array(), new_f.array()); } From 678a4e2c20642670177a5ccdc69646fc5bde5f09 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 9 Jul 2024 13:57:04 +0200 Subject: [PATCH 152/170] add weigh for SFC load balancing --- demos/FiniteVolume/advection_2d.cpp | 40 ++-- include/samurai/load_balancing_sfc_w.hpp | 289 +++++++++++++++++++++++ 2 files changed, 308 insertions(+), 21 deletions(-) create mode 100644 include/samurai/load_balancing_sfc_w.hpp diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index b23b73838..9a5692e27 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -217,8 +218,6 @@ int main(int argc, char* argv[]) { auto& app = samurai::initialize("Finite volume example for the advection equation in 2d using multiresolution", argc, argv); - Timers myTimers; - constexpr std::size_t dim = 2; using Config = samurai::MRConfig; @@ -282,20 +281,21 @@ int main(int argc, char* argv[]) rcenters.emplace_back( y_center ); radii.emplace_back( radius ); - rcenters = { 1., 1., - 1., 3., - 1., 3. }; - radii = { 0.2, 0.2, 0.2 }; + rcenters = { 0.8, 0.8, + 2., 2., + 3., 1. }; + radii = { 0.8, 0.4, 0.2 }; auto u = init(mesh, radii, rcenters); samurai::make_bc>(u, 0.); auto unp1 = samurai::make_field("unp1", mesh); - myTimers.start("make_MRAdapt_init"); auto MRadaptation = samurai::make_MRAdapt(u); + + samurai::times::timers.start("MRadaptation"); MRadaptation(mr_epsilon, mr_regularity); - myTimers.stop("make_MRAdapt_init"); + samurai::times::timers.stop("MRadaptation"); save(path, filename, u, "_init"); @@ -304,11 +304,13 @@ int main(int argc, char* argv[]) const int iof = 1; - SFC_LoadBalancer_interval balancer; + // SFC_LoadBalancer_interval balancer; + // Void_LoadBalancer balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; // Load_balancing::Diffusion balancer; + Load_balancing::SFCw balancer; std::ofstream logs; boost::mpi::communicator world; @@ -318,14 +320,14 @@ int main(int argc, char* argv[]) { if (nt % nt_loadbalance == 0 && nt > 1 ) { - myTimers.start("load-balancing"); + samurai::times::timers.start("load-balancing"); balancer.load_balance(mesh, u); - myTimers.stop("load-balancing"); + samurai::times::timers.stop("load-balancing"); } - myTimers.start("MRadaptation"); + samurai::times::timers.start("MRadaptation"); MRadaptation(mr_epsilon, mr_regularity); - myTimers.stop("MRadaptation"); + samurai::times::timers.stop("MRadaptation"); t += dt; if (t > Tf) @@ -336,15 +338,13 @@ int main(int argc, char* argv[]) std::cout << fmt::format("iteration {}: t = {}, dt = {}", nt++, t, dt) << std::endl; - myTimers.start("update_ghost_mr"); samurai::update_ghost_mr(u); - myTimers.stop("update_ghost_mr"); unp1.resize(); - myTimers.start("upwind"); + samurai::times::timers.start("upwind"); unp1 = u - dt * samurai::upwind(a, u); - myTimers.stop("upwind"); + samurai::times::timers.stop("upwind"); if (correction) { @@ -353,18 +353,16 @@ int main(int argc, char* argv[]) std::swap(u.array(), unp1.array()); - myTimers.start("I/O"); + samurai::times::timers.start("I/O"); if (t >= static_cast(nsave + 1) * dt_save || t == Tf || nt % iof == 0 ) { const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; save(path, filename, u, suffix); } - myTimers.stop("I/O"); + samurai::times::timers.stop("I/O"); } - myTimers.print(); - samurai::finalize(); return 0; } diff --git a/include/samurai/load_balancing_sfc_w.hpp b/include/samurai/load_balancing_sfc_w.hpp new file mode 100644 index 000000000..c3c2ef78b --- /dev/null +++ b/include/samurai/load_balancing_sfc_w.hpp @@ -0,0 +1,289 @@ +/** + * + * This class implements SFC-based load balancing. The difference between this class + * and the one in the load_balancing_sfc.hpp is that, here we try to use level based "weight" + * to dispatch cell between processes in addition to the 1D-SFC key. + * + * This is a try to fix the unbalanced due to the difference of load induced by the difference of + * level between two cells. With the traditional (load_balancing_sfc.hpp) strategy even if the + * load, in term of number of cells on a process, is good, unbalanced on computational time may + * appear. + * + */ +#pragma once + +#include "assertLogTrace.hpp" + +#include "load_balancing.hpp" + +namespace Load_balancing{ + + template + class SFCw : public samurai::LoadBalancer> + { + private: + + SFC_type_t _sfc; + int _ndomains; + int _rank; + + public: + + using samurai::LoadBalancer>::logs; + + SFCw() + { +#ifdef SAMURAI_WITH_MPI + boost::mpi::communicator world; + _ndomains = world.size(); + _rank = world.rank(); +#else + _ndomains = 1; + _rank = 0; +#endif + } + + inline std::string getName() const + { + return "SFCw_" + _sfc.getName() + "_LB"; + } + + /** + * Re-order cells on MPI processes based on the given SFC curve. This need to be + * called once at the beginning (unless ordering is fixed in partition mesh). + * After, load balancing will exchange data based on SFC order and thus only + * boundaries will be moved. + * + */ + template + auto reordering_impl( Mesh_t & mesh ) { + + boost::mpi::communicator world; + + // For debug + // std::ofstream logs; + // logs.open("log_" + std::to_string(_rank) + ".dat", std::ofstream::app); + logs << "# [SFCw_LoadBalancer::Morton] Reordering cells using SFC" << std::endl; + + // SFC 1D key for cells + auto sfc_keys = samurai::make_field( "keys", mesh ); + sfc_keys.fill( 0 ); + + auto flags = samurai::make_field("rank", mesh); + flags.fill( world.rank() ); + + logs << fmt::format("\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; + + SFC_key_t mink = std::numeric_limits::max(), maxk = std::numeric_limits::min(); + + samurai::for_each_cell(mesh[Mesh_t::mesh_id_t::cells], [&]( const auto & cell ) { + + // this is where things can get nasty, we expect indices to be positive values !! + xt::xtensor_fixed> ijk; + for (size_t idim = 0; idim < dim; ++idim) { + + // FIX need shift to get only positive index + assert( cell.indices( idim ) >= 0 ); + ijk( idim ) = static_cast( cell.indices( idim ) ) << ( mesh.max_level() - cell.level ); + } + + auto key = _sfc.template getKey( ijk ); + + sfc_keys[ cell ] = key; + + mink = std::min( key, mink ); + maxk = std::max( key, maxk ); + + }); + + // Key boundaries of current process - unused for now + std::vector bounds = { mink, maxk }; + logs << "\t\t> Local key bounds [" << bounds[ 0 ] << ", " << bounds[ 1 ] << "]" << std::endl; + + std::vector boundaries; + boost::mpi::all_gather( world, bounds.data(), static_cast( bounds.size() ), boundaries ); + + logs << "\t\t> Global key boundaries ["; + for(const auto & ik : boundaries ) + logs << ik << ","; + logs << "]" << std::endl; + + // Check overlap with previous/next process. Does not mean that there is no overlap, but at least between "adjacent" + // (MPI-1), (MPI+1) there is not overlap found + std::vector boundaries_new( static_cast( world.size() + 1 ) ); + + // find max value for boundaries + SFC_key_t globalMax = boundaries[ 0 ]; + for(size_t ip=0; ip( world.size() ); + boundaries_new[ 0 ] = 0; + for(size_t ip=1; ip Global key evenly distrib boundaries ["; + for(const auto & ik : boundaries_new ) + logs << ik << ","; + logs << "]" << std::endl; + + // distribute cell based on boundaries & sfc key + std::map comm; + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&](const auto & cell ) { + auto key = sfc_keys[ cell ]; + + // optimize using bisect - find proc that should have this cell + for( size_t ip=0; ip< static_cast( world.size() ); ++ip ){ + if( key >= boundaries_new[ ip ] && key < boundaries_new[ ip + 1 ] ){ + flags[ cell ] = static_cast( ip ); + + // unique list of process that should be contacted + if( comm.find( static_cast( ip ) ) == comm.end() ){ + comm[ static_cast( ip ) ] = true; + } + + break; + } + } + + }); + + return flags; + } + + template + auto load_balance_impl( Mesh_t & mesh ) + { + + boost::mpi::communicator world; + + // std::ofstream logs; + // logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs << fmt::format("\n# [SFCw_LoadBalancer::Morton] Load balancing cells ") << std::endl; + + // SFC 1D key for each cell + std::map sfc_map; + auto sfc_keys = samurai::make_field( "keys", mesh ); + sfc_keys.fill( 0 ); + + // MPI destination rank of process for each cell + auto flags = samurai::make_field("rank", mesh); + flags.fill( world.rank() ); + + // Computing weights based on maxlevel + std::vector weights( mesh.max_level() + 1 ); + for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ + weights[ ilvl ] = 1 << ( ( ilvl - mesh.min_level() ) ) ; + // weights[ ilvl ] = 1 << ( mesh.max_level() - ilvl ) ; + // weights[ ilvl ] = 1.; + logs << fmt::format("\t\t> Level {}, weight for cell : {}", ilvl, weights[ ilvl ] ) << std::endl; + } + + + logs << fmt::format("\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; + + // SFC_key_t mink = std::numeric_limits::max(), maxk = std::numeric_limits::min(); + + // compute SFC key for each cell + samurai::for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto & cell ) { + + // this is where things can get nasty, we expect indices to be positive values !! + xt::xtensor_fixed> ijk; + for (size_t idim = 0; idim < Mesh_t::dim; ++idim) { + + // FIX need shift to get only positive index + assert( cell.indices( idim ) >= 0 ); + ijk( idim ) = static_cast( cell.indices( idim ) ) << ( mesh.max_level() - cell.level ); + } + + auto key = _sfc.template getKey( ijk ); + + sfc_keys[ cell ] = key; + + if( sfc_map.find( key ) != sfc_map.end() ) { + assert( false ); + std::cerr << fmt::format("Rank # {}, Error computing SFC, index not uniq ! ", world.rank()) << std::endl; + } + + sfc_map[ key ] = cell; + + // mink = std::min( mink, key ); + // maxk = std::max( maxk, key ); + + }); + + assert( mesh.nb_cells( Mesh_t::mesh_id_t::cells ) == sfc_map.size() ); + + size_t nload_tot = 0, dc = 0; + std::vector nbLoadPerProc; + std::vector globIdx( static_cast( world.size() + 1 ) ); + std::vector globIdxNew( static_cast( world.size() + 1 ) ); + + // get load for each MPI process + size_t load = 0; + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&](const auto & cell ){ + load += weights[ cell.level ]; + }); + + boost::mpi::all_gather( world, load, nbLoadPerProc ); + + for(size_t i=0; i( world.size() ); ++i){ + globIdx[ i + 1 ] = globIdx[ i ] + nbLoadPerProc[ i ]; + nload_tot += nbLoadPerProc[ i ]; + } + dc = nload_tot / static_cast( world.size() ); + + logs << fmt::format("\t\t> Load of cells (weighted) : {}, dc : {}", load, dc) << std::endl; + + // load balanced globIdx -> new theoretical key boundaries based on number of cells per proc + globIdxNew[ 0 ] = 0; + for(size_t i=0; i( world.size() ); ++i){ + globIdxNew[ i + 1 ] = globIdxNew[ i ] + dc; + } + + { + logs << "\t\t> GlobalIdx : "; + for(const auto & i : globIdx ) + logs << i << ", "; + logs << std::endl; + + logs << "\t\t> GlobalIdx balanced : "; + for(const auto & i : globIdxNew ) + logs << i << ", "; + logs << std::endl; + + } + + size_t start = 0; + while( globIdx[ static_cast( world.rank() ) ] >= ( start + 1 ) * dc ){ + start ++; + } + + logs << "\t\t> Start @ rank " << start << std::endl; + + size_t count = globIdx[ static_cast( world.rank() ) ]; + for( auto & it : sfc_map ) { + + if( count >= ( start + 1 ) * dc ){ + start ++; + start = std::min( static_cast( world.size() - 1 ) , start ); + logs << "\t\t> Incrementing Start @ rank " << start << ", count " << count << std::endl; + } + + flags[ it.second ] = static_cast( start ); + + count += weights[ it.second.level ]; + } + + return flags; + } + + }; + +} \ No newline at end of file From e7c6e60df8797d322255cb9ccbe253188f34ae84 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Tue, 9 Jul 2024 13:57:13 +0200 Subject: [PATCH 153/170] fix assert --- include/samurai/load_balancing_sfc.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 7d84063c2..123bfefc0 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -173,7 +173,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer= 0 ); + assert( cell.indices( idim ) >= 0 ); ijk( idim ) = static_cast( cell.indices( idim ) ) << ( mesh.max_level() - cell.level ); } @@ -209,7 +209,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer( world.size() ); - // load balanced globIdx + // load balanced globIdx -> new theoretical key boundaries based on number of cells per proc globIdxNew[ 0 ] = 0; for(size_t i=0; i( world.size() ); ++i){ globIdxNew[ i + 1 ] = globIdxNew[ i ] + dc; From 2cc21231e0514b0650178410ef2e79d074126529 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 10 Jul 2024 17:27:10 +0200 Subject: [PATCH 154/170] add unbalance boolean criteria to trigger load balancing --- demos/FiniteVolume/advection_2d.cpp | 10 ++++++--- include/samurai/load_balancing.hpp | 11 +++++++++- include/samurai/load_balancing_sfc.hpp | 29 +++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 9a5692e27..7cdf01a64 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -304,13 +304,13 @@ int main(int argc, char* argv[]) const int iof = 1; - // SFC_LoadBalancer_interval balancer; + SFC_LoadBalancer_interval balancer; // Void_LoadBalancer balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; // Load_balancing::Diffusion balancer; - Load_balancing::SFCw balancer; + // Load_balancing::SFCw balancer; std::ofstream logs; boost::mpi::communicator world; @@ -318,7 +318,11 @@ int main(int argc, char* argv[]) while (t != Tf) { - if (nt % nt_loadbalance == 0 && nt > 1 ) + bool reqBalance = balancer.require_balance( mesh ); + + if( reqBalance ) std::cerr << "\t> Load Balancing required !!! " << std::endl; + + if ( ( nt % nt_loadbalance == 0 || reqBalance ) && nt > 1 ) { samurai::times::timers.start("load-balancing"); balancer.load_balance(mesh, u); diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index 095609f72..cbb3be14d 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -687,7 +687,7 @@ namespace samurai logs << "\t\t>[Load_balancer::update_mesh] Comm required with processes : ["; for( const auto & it : comm ) - logs << it.first << fmt::format(" ({} cells),", payload_size[it.first]); + logs << it.first << fmt::format(" ({} cells),", payload_size[ static_cast( it.first ) ]); logs << "]" << std::endl; std::vector req_send( static_cast( world.size() ), 0 ), req_recv( static_cast( world.size() ), 0 ); @@ -791,6 +791,15 @@ namespace samurai nloadbalancing += 1; } + /* + * Call balancer function that evaluate if load balancing is required or not based on balncer internal strategy + * + */ + template + bool require_balance( Mesh_t & mesh ) { + return static_cast(this)->require_balance_impl( mesh ); + } + /** * This function MUST be used for debug or analysis purposes of load balancing strategy, * it involves a lots of MPI communications. diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 123bfefc0..844d1e1ea 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -13,7 +13,7 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer + bool require_balance_impl( Mesh_t & mesh ) { + + boost::mpi::communicator world; + + logs << fmt::format("\n# [SFC_LoadBalancer_interval::Morton] required_balance_impl ") << std::endl; + + double nbCells_tot = 0; + std::vector nbCellsPerProc; + boost::mpi::all_gather( world, static_cast( mesh.nb_cells( Mesh_t::mesh_id_t::cells ) ), nbCellsPerProc ); + + for(size_t ip=0; ip ( world.size() ); + + for(size_t ip=0; ip _unbalance_threshold ) return true; + } + + return false; + } + template auto load_balance_impl( Mesh_t & mesh ) { From 868b25d603ec870fa40252a4ecfc75625354dcb4 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 11 Jul 2024 11:48:12 +0200 Subject: [PATCH 155/170] update sfc w require_loadbalancing --- demos/FiniteVolume/advection_2d.cpp | 4 +- include/samurai/load_balancing_sfc_w.hpp | 71 +++++++++++++++++++++--- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 7cdf01a64..51454a5e2 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -322,7 +322,9 @@ int main(int argc, char* argv[]) if( reqBalance ) std::cerr << "\t> Load Balancing required !!! " << std::endl; - if ( ( nt % nt_loadbalance == 0 || reqBalance ) && nt > 1 ) + // if ( ( nt % nt_loadbalance == 0 || reqBalance ) && nt > 1 ) + if ( ( nt % nt_loadbalance == 0 ) && nt > 1 ) + // if ( reqBalance && nt > 1 ) { samurai::times::timers.start("load-balancing"); balancer.load_balance(mesh, u); diff --git a/include/samurai/load_balancing_sfc_w.hpp b/include/samurai/load_balancing_sfc_w.hpp index c3c2ef78b..14c087b00 100644 --- a/include/samurai/load_balancing_sfc_w.hpp +++ b/include/samurai/load_balancing_sfc_w.hpp @@ -27,6 +27,8 @@ namespace Load_balancing{ int _ndomains; int _rank; + const double _unbalance_threshold = 0.05; // 5 % + public: using samurai::LoadBalancer>::logs; @@ -48,6 +50,59 @@ namespace Load_balancing{ return "SFCw_" + _sfc.getName() + "_LB"; } + /** + * Compute weights for cell at each level. We expect that small cell (high level) + * cost more computational power that largest cell (low level). But this is impacted + * by numerical scheme used. As default value, we use power of two starting from levelmin + * (low level). + */ + std::vector getWeights( size_t levelmin, size_t levelmax ) const { + // Computing weights based on maxlevel + std::vector weights( levelmax + 1 ); + + for( size_t ilvl=levelmin; ilvl<=levelmax; ++ilvl ){ + weights[ ilvl ] = 1 << ( ( ilvl - levelmin ) ) ; // prioritize small cell + // weights[ ilvl ] = 1 << ( mesh.max_level() - ilvl ) ; // prioritize large cell + // weights[ ilvl ] = 1.; // all equal + logs << fmt::format("\t\t> Level {}, weight for cell : {}", ilvl, weights[ ilvl ] ) << std::endl; + } + + return weights; + } + + template + bool require_balance_impl( Mesh_t & mesh ) { + + boost::mpi::communicator world; + + logs << fmt::format("\n# [SFCw_LoadBalancer::Morton] required_balance_impl ") << std::endl; + + std::vector weights = getWeights( mesh.min_level(), mesh.max_level() ); + + double load = 0; + samurai::for_each_cell( mesh[Mesh_t::mesh_id_t::cells], [&](const auto & cell ){ + load += weights[ cell.level ]; + }); + + std::vector nbLoadPerProc; + boost::mpi::all_gather( world, load, nbLoadPerProc ); + + double load_tot = 0.; + for(size_t i=0; i( world.size() ); + + for(size_t ip=0; ip _unbalance_threshold ) return true; + } + + return false; + } + /** * Re-order cells on MPI processes based on the given SFC curve. This need to be * called once at the beginning (unless ordering is fixed in partition mesh). @@ -177,13 +232,15 @@ namespace Load_balancing{ flags.fill( world.rank() ); // Computing weights based on maxlevel - std::vector weights( mesh.max_level() + 1 ); - for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ - weights[ ilvl ] = 1 << ( ( ilvl - mesh.min_level() ) ) ; - // weights[ ilvl ] = 1 << ( mesh.max_level() - ilvl ) ; - // weights[ ilvl ] = 1.; - logs << fmt::format("\t\t> Level {}, weight for cell : {}", ilvl, weights[ ilvl ] ) << std::endl; - } + // std::vector weights( mesh.max_level() + 1 ); + // for( size_t ilvl=mesh.min_level(); ilvl<=mesh.max_level(); ++ilvl ){ + // weights[ ilvl ] = 1 << ( ( ilvl - mesh.min_level() ) ) ; + // // weights[ ilvl ] = 1 << ( mesh.max_level() - ilvl ) ; + // // weights[ ilvl ] = 1.; + // logs << fmt::format("\t\t> Level {}, weight for cell : {}", ilvl, weights[ ilvl ] ) << std::endl; + // } + + std::vector weights = getWeights( mesh.min_level(), mesh.max_level() ); logs << fmt::format("\t> Computing SFC ({}) 1D indices ( cell ) ... ", _sfc.getName() ) << std::endl; From 5d663b9425ea28ed3505449c044a5860cfaf6f69 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 12 Jul 2024 09:37:00 +0200 Subject: [PATCH 156/170] fix const compite error --- include/samurai/load_balancing_sfc_w.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/samurai/load_balancing_sfc_w.hpp b/include/samurai/load_balancing_sfc_w.hpp index 14c087b00..ba0aa395b 100644 --- a/include/samurai/load_balancing_sfc_w.hpp +++ b/include/samurai/load_balancing_sfc_w.hpp @@ -56,7 +56,7 @@ namespace Load_balancing{ * by numerical scheme used. As default value, we use power of two starting from levelmin * (low level). */ - std::vector getWeights( size_t levelmin, size_t levelmax ) const { + std::vector getWeights( size_t levelmin, size_t levelmax ) { // Computing weights based on maxlevel std::vector weights( levelmax + 1 ); From 855daecae5736f6b3d7f8ee95eff731eacba9e98 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Fri, 12 Jul 2024 11:45:36 +0200 Subject: [PATCH 157/170] update diff LB --- include/samurai/load_balancing_diffusion.hpp | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 4ab98539a..610cad769 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -16,6 +16,8 @@ namespace Load_balancing{ int _ndomains; int _rank; + const double _unbalance_threshold = 0.05; // 5 % + public: Diffusion() { @@ -32,6 +34,33 @@ namespace Load_balancing{ inline std::string getName() const { return "diffusion"; } + template + bool require_balance_impl( Mesh_t & mesh ) { + + boost::mpi::communicator world; + + logs << fmt::format("\n# [Diffusion_LoadBalancer] required_balance_impl ") << std::endl; + + double nbCells_tot = 0; + std::vector nbCellsPerProc; + boost::mpi::all_gather( world, static_cast( mesh.nb_cells( Mesh_t::mesh_id_t::cells ) ), nbCellsPerProc ); + + for(size_t ip=0; ip ( world.size() ); + + for(size_t ip=0; ip _unbalance_threshold ) return true; + } + + return false; + } + template auto reordering_impl( Mesh_t & mesh ) { auto flags = samurai::make_field("diffusion_flag", mesh); From 04bb9800f581578a9022b9146ddb45bebd0dd3ff Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 22 Jul 2024 09:19:38 +0200 Subject: [PATCH 158/170] update intro + config --- include/samurai/samurai.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/samurai/samurai.hpp b/include/samurai/samurai.hpp index 6843d9b20..4d13df6ac 100644 --- a/include/samurai/samurai.hpp +++ b/include/samurai/samurai.hpp @@ -31,6 +31,12 @@ namespace samurai #ifdef SAMURAI_WITH_MPI MPI_Init(&argc, &argv); + + boost::mpi::communicator world; + + rank = world.rank(); + nproc = world.size(); + mpi_tag = "ON"; #endif times::timers.start("total runtime"); return app; From 0de0541b3324f5b0b72161f0c9c12ff73edaecd7 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 22 Jul 2024 09:20:10 +0200 Subject: [PATCH 159/170] test different cell weights --- include/samurai/load_balancing_sfc_w.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/samurai/load_balancing_sfc_w.hpp b/include/samurai/load_balancing_sfc_w.hpp index ba0aa395b..b7965671c 100644 --- a/include/samurai/load_balancing_sfc_w.hpp +++ b/include/samurai/load_balancing_sfc_w.hpp @@ -61,8 +61,9 @@ namespace Load_balancing{ std::vector weights( levelmax + 1 ); for( size_t ilvl=levelmin; ilvl<=levelmax; ++ilvl ){ - weights[ ilvl ] = 1 << ( ( ilvl - levelmin ) ) ; // prioritize small cell - // weights[ ilvl ] = 1 << ( mesh.max_level() - ilvl ) ; // prioritize large cell + // weights[ ilvl ] = ( 1 << ( ilvl - levelmin ) ); // prioritize small cell + weights[ ilvl ] = 1 << (ilvl * ilvl); + // weights[ ilvl ] = 1 << ( levelmax - ilvl ) ; // prioritize large cell // weights[ ilvl ] = 1.; // all equal logs << fmt::format("\t\t> Level {}, weight for cell : {}", ilvl, weights[ ilvl ] ) << std::endl; } From 96fe005522f9444118b6ff389915d7ac93f5e775 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Mon, 22 Jul 2024 09:20:56 +0200 Subject: [PATCH 160/170] fix: add missing func to update new lb auto trigger --- include/samurai/load_balancing_force.hpp | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/include/samurai/load_balancing_force.hpp b/include/samurai/load_balancing_force.hpp index 3465f834e..34f64cc42 100644 --- a/include/samurai/load_balancing_force.hpp +++ b/include/samurai/load_balancing_force.hpp @@ -12,6 +12,8 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer double getSurfaceOrVolume( Mesh_t & mesh ) const { @@ -45,6 +47,33 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer + bool require_balance_impl( Mesh_t & mesh ) { + + boost::mpi::communicator world; + + // logs << fmt::format("\n# [SFC_LoadBalancer_interval::Morton] required_balance_impl ") << std::endl; + + double nbCells_tot = 0; + std::vector nbCellsPerProc; + boost::mpi::all_gather( world, static_cast( mesh.nb_cells( Mesh_t::mesh_id_t::cells ) ), nbCellsPerProc ); + + for(size_t ip=0; ip ( world.size() ); + + for(size_t ip=0; ip _unbalance_threshold ) return true; + } + + return false; + } + template auto reordering_impl( Mesh_t & mesh ){ auto flags = samurai::make_field("rank", mesh); @@ -132,7 +161,7 @@ class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer("rank", mesh); flags.fill( world.rank() ); - constexpr auto fdist = samurai::Distance_t::L2; + constexpr auto fdist = samurai::Distance_t::L1; double currentSV = getSurfaceOrVolume( mesh ); From d750d4c2c392ca27627b7baff9f1a57607ef3e62 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 24 Jul 2024 09:31:27 +0200 Subject: [PATCH 161/170] fix void load balancer --- include/samurai/load_balancing_void.hpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/include/samurai/load_balancing_void.hpp b/include/samurai/load_balancing_void.hpp index 7d2cf4c57..75de08263 100644 --- a/include/samurai/load_balancing_void.hpp +++ b/include/samurai/load_balancing_void.hpp @@ -30,10 +30,21 @@ class Void_LoadBalancer: public samurai::LoadBalancer> { inline std::string getName() const { return "Void_LB"; } + template + bool require_balance_impl( Mesh_t & mesh ) { return false; } + template - Mesh_t reordering_impl( Mesh_t & mesh ) { return mesh; } + auto reordering_impl( Mesh_t & mesh ) { + auto flags = samurai::make_field("balancing_flags", mesh); + flags.fill( _rank ); + return flags; + } template - Mesh_t load_balance_impl( Mesh_t & mesh ){ return mesh; } + auto load_balance_impl( Mesh_t & mesh ){ + auto flags = samurai::make_field("balancing_flags", mesh); + flags.fill( _rank ); + return flags; + } }; \ No newline at end of file From 22e360315353e22c4c92522b0ba929b3335ea69d Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Wed, 24 Jul 2024 09:32:31 +0200 Subject: [PATCH 162/170] add test with ponio --- demos/FiniteVolume/CMakeLists.txt | 8 + .../FiniteVolume/linear_convection_ponio.cpp | 278 ++++++++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 demos/FiniteVolume/linear_convection_ponio.cpp diff --git a/demos/FiniteVolume/CMakeLists.txt b/demos/FiniteVolume/CMakeLists.txt index 1418a705b..829c86d0e 100644 --- a/demos/FiniteVolume/CMakeLists.txt +++ b/demos/FiniteVolume/CMakeLists.txt @@ -30,6 +30,14 @@ endif() add_executable(finite-volume-linear-convection linear_convection.cpp) target_link_libraries(finite-volume-linear-convection samurai CLI11::CLI11 ${MPI_LIBRARIES}) +# Test with Ponio (required CXX 20) +find_package(ponio CONFIG REQUIRED) + +add_executable(finite-volume-linear-convection-ponio linear_convection_ponio.cpp) +target_include_directories(finite-volume-linear-convection-ponio PUBLIC /Users/strafella/work/ponio/ponio/include/) +target_link_libraries(finite-volume-linear-convection-ponio samurai CLI11::CLI11 ${MPI_LIBRARIES} ponio::ponio ponio::project_options ponio::project_warnings) + + add_executable(finite-volume-amr-burgers-hat AMR_Burgers_Hat.cpp) target_link_libraries(finite-volume-amr-burgers-hat samurai CLI11::CLI11) diff --git a/demos/FiniteVolume/linear_convection_ponio.cpp b/demos/FiniteVolume/linear_convection_ponio.cpp new file mode 100644 index 000000000..2e4594c27 --- /dev/null +++ b/demos/FiniteVolume/linear_convection_ponio.cpp @@ -0,0 +1,278 @@ +// Copyright 2018-2024 the samurai's authors +// SPDX-License-Identifier: BSD-3-Clause +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ponio/runge_kutta.hpp" +#include "ponio/solver.hpp" + +#ifdef WITH_STATS +#include "samurai/statistics.hpp" +#endif + +#include +namespace fs = std::filesystem; + +template +double exact_solution(xt::xtensor_fixed> coords, double t) +{ + const double a = 1; + const double b = 0; + const double& x = coords(0); + return (a * x + b) / (a * t + 1); +} + +template +void save(const fs::path& path, const std::string& filename, Field& u, const std::string& suffix = "") +{ + auto mesh = u.mesh(); + u.name() = "u"; + auto level_ = samurai::make_field("level", mesh); + + if (!fs::exists(path)) + { + fs::create_directory(path); + } + + samurai::for_each_cell(mesh, + [&](const auto& cell) + { + level_[cell] = cell.level; + }); + + samurai::save(path, fmt::format("{}{}", filename, suffix), mesh, u, level_); +} + +int main(int argc, char* argv[]) +{ + samurai::initialize(argc, argv); + + static constexpr std::size_t dim = 2; + using Config = samurai::MRConfig; + using Box = samurai::Box; + using point_t = typename Box::point_t; + + std::cout << "------------------------- Linear convection -------------------------" << std::endl; + + //--------------------// + // Program parameters // + //--------------------// + boost::mpi::communicator world; + + // Simulation parameters + double left_box = 0; + double right_box = 4; + std::string init_sol = "hat"; + + // Time integration + double Tf = 3; + double dt = 0; + double cfl = 0.95; + + // Multiresolution parameters + std::size_t min_level = 0; + std::size_t max_level = dim == 1 ? 5 : 3; + double mr_epsilon = 1e-4; // Threshold used by multiresolution + double mr_regularity = 1.; // Regularity guess for multiresolution + + // Output parameters + fs::path path = fs::current_path(); + std::string filename = "linear_convection_" + std::to_string(dim) + "D"; + std::size_t nfiles = 0; + std::size_t nt_loadbalance = 10; + + CLI::App app{"Finite volume example for the heat equation in 1d"}; + app.add_option("--left", left_box, "The left border of the box")->capture_default_str()->group("Simulation parameters"); + app.add_option("--right", right_box, "The right border of the box")->capture_default_str()->group("Simulation parameters"); + app.add_option("--init-sol", init_sol, "Initial solution: hat/linear/bands")->capture_default_str()->group("Simulation parameters"); + app.add_option("--Tf", Tf, "Final time")->capture_default_str()->group("Simulation parameters"); + app.add_option("--dt", dt, "Time step")->capture_default_str()->group("Simulation parameters"); + app.add_option("--cfl", cfl, "The CFL")->capture_default_str()->group("Simulation parameters"); + app.add_option("--min-level", min_level, "Minimum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--max-level", max_level, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--nt-loadbalance", nt_loadbalance, "Maximum level of the multiresolution")->capture_default_str()->group("Multiresolution"); + app.add_option("--mr-eps", mr_epsilon, "The epsilon used by the multiresolution to adapt the mesh") + ->capture_default_str() + ->group("Multiresolution"); + app.add_option("--mr-reg", + mr_regularity, + "The regularity criteria used by the multiresolution to " + "adapt the mesh") + ->capture_default_str() + ->group("Multiresolution"); + app.add_option("--path", path, "Output path")->capture_default_str()->group("Ouput"); + app.add_option("--filename", filename, "File name prefix")->capture_default_str()->group("Ouput"); + app.add_option("--nfiles", nfiles, "Number of output files")->capture_default_str()->group("Ouput"); + app.allow_extras(); + CLI11_PARSE(app, argc, argv); + + //--------------------// + // Problem definition // + //--------------------// + + point_t box_corner1, box_corner2; + box_corner1.fill(left_box); + box_corner2.fill(right_box); + Box box(box_corner1, box_corner2); + std::array periodic; + periodic.fill(false); + + samurai::times::timers.start("init"); + samurai::MRMesh mesh{box, min_level, max_level, periodic}; + + // Initial solution + auto u = samurai::make_field("u", + mesh, + [&](const auto& coords) + { + if constexpr (dim == 1) + { + auto& x = coords(0); + return (x >= -0.8 && x <= -0.3) ? 1. : 0.; + } + else if ( dim == 2 ) + { + auto& x = coords(0); + auto& y = coords(1); + return (x <= 0.8 && x >= 0.3 && y >= 0.3 && y <= 0.8) ? 1. : 0.; + }else { + auto & x = coords( 0 ); + auto & y = coords( 1 ); + auto & z = coords( 2 ); + return (x <= 0.8 && x >= 0.3 && y >= 0.3 && y <= 0.8 && z >= 0.3 && z <= 0.8 ) ? 1. : 0.; + } + + }); + samurai::times::timers.stop("init"); + + samurai::make_bc>(u, 0.); + + // Convection operator + samurai::VelocityVector velocity; + velocity.fill(1); + + // origin weno5 + auto conv = samurai::make_convection_weno5(velocity); + + auto ponio_f = [&]([[maybe_unused]]double t, auto && u){ + samurai::make_bc>(u, 0.); + samurai::update_ghost_mr( u ); + return - conv( u ); + }; + + // SFC_LoadBalancer_interval balancer; + // Load_balancing::Life balancer; + // Void_LoadBalancer balancer; + Diffusion_LoadBalancer_cell balancer; + // Diffusion_LoadBalancer_interval balancer; + // Load_balancing::Diffusion balancer; + + //--------------------// + // Time iteration // + //--------------------// + + if (dt == 0) + { + double dx = samurai::cell_length(max_level); + auto a = xt::abs(velocity); + double sum_velocities = xt::sum(xt::abs(velocity))(); + dt = cfl * dx / sum_velocities; + } + + auto sol_range = ponio::make_solver_range( ponio_f, ponio::runge_kutta::rk_33(), u, {0.0, Tf}, dt ); + + auto it = sol_range.begin(); + + auto MRadaptation = samurai::make_MRAdapt( it->state ); + + samurai::times::timers.start("MRadaptation"); + MRadaptation(mr_epsilon, mr_regularity); + samurai::times::timers.stop("MRadaptation"); + + double dt_save = nfiles == 0 ? dt : Tf / static_cast(nfiles); + std::size_t nsave = 0, nt = 0; + if (nfiles != 1) + { + std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; + save(path, filename, it->state, suffix); + } + + samurai::times::timers.start("tloop"); + + while ( it->time != Tf) + { + + if ( balancer.require_balance( mesh ) ) + { + samurai::times::timers.start("tloop.lb:"+balancer.getName()); + balancer.load_balance(mesh, it->state); + samurai::times::timers.stop("tloop.lb:"+balancer.getName()); + } + std::cout << fmt::format("iteration {}: t = {:.2f}, dt = {}", nt++, it->time, it->time_step) << std::endl; + + // Mesh adaptation + samurai::times::timers.start("tloop.MRadaptation"); + MRadaptation(mr_epsilon, mr_regularity); + samurai::times::timers.stop("tloop.MRadaptation"); + + samurai::times::timers.start("tloop.ugm"); + samurai::update_ghost_mr( it->state ); + samurai::times::timers.stop("tloop.ugm"); + + samurai::times::timers.start("tloop.resize_fill"); + for ( auto& ki : it.meth.kis ) + { + ki.resize(); + ki.fill( 0. ); + } + samurai::times::timers.stop("tloop.resize_fill"); + + samurai::times::timers.start("tloop.scheme"); + ++it; + samurai::times::timers.stop("tloop.scheme"); + + // Save the result + samurai::times::timers.start("tloop.io"); + if (nfiles == 0 || it->time >= static_cast(nsave + 1) * dt_save || it->time == Tf) + { + if (nfiles != 1) + { + std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; + save(path, filename, it->state, suffix); + } + else + { + save(path, filename, it->state); + } + } + samurai::times::timers.stop("tloop.io"); + + } + samurai::times::timers.stop("tloop"); + + if constexpr (dim == 1) + { + std::cout << std::endl; + std::cout << "Run the following command to view the results:" << std::endl; + std::cout << "python <>/python/read_mesh.py " << filename << "_ite_ --field u level --start 0 --end " << nsave + << std::endl; + } + + samurai::finalize(); + return 0; +} From c4b5d20ca298b785153750468c59cf99895c6b30 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 25 Jul 2024 10:44:55 +0200 Subject: [PATCH 163/170] fix warning --- include/samurai/load_balancing_void.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/samurai/load_balancing_void.hpp b/include/samurai/load_balancing_void.hpp index 75de08263..65477f5b8 100644 --- a/include/samurai/load_balancing_void.hpp +++ b/include/samurai/load_balancing_void.hpp @@ -31,7 +31,7 @@ class Void_LoadBalancer: public samurai::LoadBalancer> { inline std::string getName() const { return "Void_LB"; } template - bool require_balance_impl( Mesh_t & mesh ) { return false; } + bool require_balance_impl( [[maybe_unused]] Mesh_t & mesh ) { return false; } template auto reordering_impl( Mesh_t & mesh ) { From 42c5216ef544708e5d531446ef7012a548bcfea5 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 25 Jul 2024 10:45:12 +0200 Subject: [PATCH 164/170] fix warning --- include/samurai/load_balancing_life.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/samurai/load_balancing_life.hpp b/include/samurai/load_balancing_life.hpp index d06ad0c38..41f76a7f3 100644 --- a/include/samurai/load_balancing_life.hpp +++ b/include/samurai/load_balancing_life.hpp @@ -44,11 +44,11 @@ namespace Load_balancing{ auto load_balance_impl( Mesh_t & mesh ){ using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; - using CellList_t = typename Mesh_t::cl_type; - using mesh_id_t = typename Mesh_t::mesh_id_t; + // using CellList_t = typename Mesh_t::cl_type; + // using mesh_id_t = typename Mesh_t::mesh_id_t; - using Coord_t = xt::xtensor_fixed>; - using Stencil = xt::xtensor_fixed>; + // using Coord_t = xt::xtensor_fixed>; + // using Stencil = xt::xtensor_fixed>; boost::mpi::communicator world; From be71678e999e8f481c9093e31a988d409ae7dcf2 Mon Sep 17 00:00:00 2001 From: "L. Strafella" Date: Thu, 25 Jul 2024 10:45:47 +0200 Subject: [PATCH 165/170] impl new approach --- include/samurai/load_balancing_force.hpp | 583 +++++++++++++++++------ 1 file changed, 433 insertions(+), 150 deletions(-) diff --git a/include/samurai/load_balancing_force.hpp b/include/samurai/load_balancing_force.hpp index 34f64cc42..e39ad69f5 100644 --- a/include/samurai/load_balancing_force.hpp +++ b/include/samurai/load_balancing_force.hpp @@ -3,224 +3,507 @@ #include #include "load_balancing.hpp" -template -class Diffusion_LoadBalancer_cell : public samurai::LoadBalancer> { +namespace Load_balancing{ - using Coord_t = xt::xtensor_fixed>; + class GlobalCriteria : public samurai::LoadBalancer { - private: - int _ndomains; - int _rank; + private: + int _ndomains; + int _rank; - const double _unbalance_threshold = 0.05; // 5 % + const double _unbalance_threshold = 0.05; // 5 % - template - double getSurfaceOrVolume( Mesh_t & mesh ) const { + template + double getSurfaceOrVolume( Mesh_t & mesh ) const { - double s_ = 0.; - for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ - - double s = 1.; + double s_ = 0.; + for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ + + double s = 1.; - for(size_t idim=0; idim - bool require_balance_impl( Mesh_t & mesh ) { + template + bool require_balance_impl( Mesh_t & mesh ) { - boost::mpi::communicator world; + boost::mpi::communicator world; - // logs << fmt::format("\n# [SFC_LoadBalancer_interval::Morton] required_balance_impl ") << std::endl; + // logs << fmt::format("\n# [SFC_LoadBalancer_interval::Morton] required_balance_impl ") << std::endl; - double nbCells_tot = 0; - std::vector nbCellsPerProc; - boost::mpi::all_gather( world, static_cast( mesh.nb_cells( Mesh_t::mesh_id_t::cells ) ), nbCellsPerProc ); + double nbCells_tot = 0; + std::vector nbCellsPerProc; + boost::mpi::all_gather( world, static_cast( mesh.nb_cells( Mesh_t::mesh_id_t::cells ) ), nbCellsPerProc ); - for(size_t ip=0; ip ( world.size() ); - // no weight while computing load - double dc = nbCells_tot / static_cast ( world.size() ); + for(size_t ip=0; ip _unbalance_threshold ) return true; + } - if( diff > _unbalance_threshold ) return true; + return false; } - return false; - } + template + auto reordering_impl( Mesh_t & mesh ){ + auto flags = samurai::make_field("rank", mesh); + flags.fill( _rank ); + return flags; + } - template - auto reordering_impl( Mesh_t & mesh ){ - auto flags = samurai::make_field("rank", mesh); - flags.fill( _rank ); - return flags; - } + template + auto load_balance_impl( Mesh_t & mesh ){ - template - auto load_balance_impl( Mesh_t & mesh ){ + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + using Coord_t = xt::xtensor_fixed>; - using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + boost::mpi::communicator world; - boost::mpi::communicator world; + // For debug purpose + std::ofstream logs; + logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); + logs << "# New load balancing" << std::endl; - // For debug purpose - std::ofstream logs; - logs.open( "log_" + std::to_string( world.rank() ) + ".dat", std::ofstream::app ); - logs << "# New load balancing" << std::endl; + // give access to rank & mesh of neighbour + std::vector & neighbourhood = mesh.mpi_neighbourhood(); + + std::size_t n_neighbours = neighbourhood.size(); + + std::vector loads; + double my_load = static_cast( samurai::cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load, loads ); + + // get the load to neighbours (geometrical neighbour) with 5 iterations max + std::vector fluxes = samurai::cmptFluxes( mesh, 1 ); + + { + logs << "load : " << my_load << std::endl; + logs << "nneighbours : " << n_neighbours << std::endl; + logs << "neighbours : "; + for( size_t in=0; in & neighbourhood = mesh.mpi_neighbourhood(); - - std::size_t n_neighbours = neighbourhood.size(); + // Interface for each neighbour: std::vector + auto interface = samurai::_computeCartesianInterface( mesh ); - std::vector loads; - double my_load = static_cast( samurai::cmptLoad( mesh ) ); - boost::mpi::all_gather( world, my_load, loads ); + // Invalidate fluxes with non adjacents neighbours, should not happen or might happen if load balancing exchange does not consider some + // direction. + std::vector ncells_interface ( n_neighbours, 0 ); + for(size_t ni=0; ni fluxes = samurai::cmptFluxes( mesh, 1 ); + if( ncells_interface[ ni ] == 0 ) fluxes[ ni ] = 0; + } + + // bary center of current mesh - no weight on cells + // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = samurai::_cmpCellBarycenter( mesh ); + logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; + + // std::vector barycenter_interface_neighbours( n_neighbours ); + std::vector barycenter_neighbours( n_neighbours ); + std::vector sv( n_neighbours ); + + for(size_t nbi=0; nbi( interface[ nbi ] ); + barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh ); + + // surface or volume depending on Mesh_t::dim + sv[ nbi ] = getSurfaceOrVolume( neighbourhood[ nbi ].mesh ); + + // debug + auto s_ = fmt::format( "Barycenter neighbour # {}: ({}, {})", neighbourhood[ nbi ].rank, + barycenter_neighbours[ nbi ]( 0 ), + barycenter_neighbours[ nbi ]( 1 ) ); + + logs << s_ << std::endl; + } + + // build map of interval that needs to be sent. Warning, it does not work with classical std::map !! + auto flags = samurai::make_field("rank", mesh); + flags.fill( world.rank() ); + + constexpr auto fdist = samurai::Distance_t::L1; + + double currentSV = getSurfaceOrVolume( mesh ); + + std::vector mload( static_cast( world.size() ), 0 ); + for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ + + auto cc = cell.center(); + + // process that might get the interval + int winner_id = -1; + + // double winner_dist = std::numeric_limits::max(); + // double winner_dist = samurai::getDistance( cell, barycenter ) / loads[ world.rank() ]; + + // double coeff_current = currentSV / loads[ static_cast( world.rank() ) ]; + double coeff_current = 1.; // loads[ static_cast( world.rank() ) ]; + double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; + // logs << fmt::format("\t\t> Cell : ({},{}), current mesh dist : {}", cc(0), cc(1), winner_dist ) << std::endl; + + // select the neighbour + for( std::size_t ni=0; ni( neighbourhood[ ni ].rank ); + + if( fluxes[ ni ] >= 0 ) continue; // skip neighbour that will recv + + // this might fix ilots but require neighbour update + // double dist = std::min( distance_inf( mid_point, barycenter_interface_neighbours[ ni ] ), + // distance_inf( mid_point, barycenter_neighbours[ ni ] ) ); + + // double dist = samurai::getDistance( cell, barycenter_interface_neighbours[ ni ] ) / loads[ neighbour_rank ]; + // double coeff = sv[ ni ] / ( loads[ neighbour_rank ] ) ; // / sv[ ni ]; + double coeff = 1.; // loads[ neighbour_rank ] ; // mload[ neighbour_rank ]; + double dist = samurai::getDistance( cell.center(), barycenter_neighbours[ ni ] ) * coeff; + + // logs << fmt::format("\t\t\t> Dist to neighbour {} : {}", neighbour_rank, dist ) << std::endl; + + if( dist < winner_dist && ncells_interface[ ni ] > 0 ){ + winner_id = static_cast( ni ); + winner_dist = dist; + + mload[ neighbour_rank ] += 1; + + // if( mload[ neighbour_rank ] >= ( - fluxes[ ni ] ) ) fluxes[ ni ] = 0; + } + + } + + // logs << fmt::format("\t\t\t> Cell given to process #{}", neighbourhood[ static_cast( winner_id ) ].rank ) << std::endl; + + if( winner_id >= 0 ){ + assert( winner_id < neighbourhood.size() ); + flags[ cell ] = neighbourhood[ static_cast( winner_id ) ].rank; + } + + }); + + return flags; + } + + template + xt::xtensor_fixed> _barycenterWithFlags( const Mesh_t & mesh, const Field_t & flags ) { - logs << "load : " << my_load << std::endl; - logs << "nneighbours : " << n_neighbours << std::endl; - logs << "neighbours : "; - for( size_t in=0; in>; + + Coord_t bary; + bary.fill(0.); + + double wght_tot = 0.; + samurai::for_each_cell(mesh, [&, this]( const auto & cell ) { + double wght = 1.; + + if constexpr( w == samurai::Weight::OnSmall ){ + wght = 1. / static_cast( 1 << (mesh.max_level() - cell.level) ); + } + + if constexpr( w == samurai::Weight::OnLarge ){ + wght = 1. / static_cast( 1 << cell.level ); + } + + if( flags[ cell ] == this->_rank ) { + auto cc = cell.center(); + + for (size_t idim = 0; idim < dim; ++idim) + { + bary(idim) += cc(idim) * wght; + } + + wght_tot += wght; + } + + }); + + wght_tot = std::max(wght_tot, 1e-12); + for (size_t idim = 0; idim < dim; ++idim) + { + bary(idim) /= wght_tot; + } + + return bary; } - // Interface for each neighbour: std::vector - auto interface = samurai::_computeCartesianInterface( mesh ); + template + xt::xtensor_fixed> _barycenterWithMeshWithFlags( const Mesh_t & mesh, const Field_t & flags, int rank, const Mesh_t & otherMesh ) + { + using Coord_t = xt::xtensor_fixed>; + + Coord_t bary; + bary.fill(0.); + + double wght_tot = 0.; + samurai::for_each_cell(mesh, [&]( const auto & cell ) { + double wght = 1.; + + if constexpr( w == samurai::Weight::OnSmall ){ + wght = 1. / static_cast( 1 << (mesh.max_level() - cell.level) ); + } + + if constexpr( w == samurai::Weight::OnLarge ){ + wght = 1. / static_cast( 1 << cell.level ); + } + + if( flags[ cell ] == rank ) { + auto cc = cell.center(); + + for (size_t idim = 0; idim < dim; ++idim) + { + bary(idim) += cc(idim) * wght; + } + + wght_tot += wght; + } - // Invalidate fluxes with non adjacents neighbours, should not happen or might happen if load balancing exchange does not consider some - // direction. - std::vector ncells_interface ( n_neighbours, 0 ); - for(size_t ni=0; ni( 1 << (mesh.max_level() - cell.level) ); + } + + if constexpr( w == samurai::Weight::OnLarge ){ + wght = 1. / static_cast( 1 << cell.level ); + } + + auto cc = cell.center(); + + for (size_t idim = 0; idim < dim; ++idim) + { + bary(idim) += cc(idim) * wght; + } + + wght_tot += wght; + + }); + + wght_tot = std::max(wght_tot, 1e-12); + for (size_t idim = 0; idim < dim; ++idim) + { + bary(idim) /= wght_tot; + } + + return bary; } - // bary center of current mesh - no weight on cells - // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); - Coord_t barycenter = samurai::_cmpCellBarycenter( mesh ); - logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; + template + auto load_balance_impl2( Mesh_t & mesh ){ + + using mpi_subdomain_t = typename Mesh_t::mpi_subdomain_t; + using Coord_t = xt::xtensor_fixed>; - // std::vector barycenter_interface_neighbours( n_neighbours ); - std::vector barycenter_neighbours( n_neighbours ); - std::vector sv( n_neighbours ); + boost::mpi::communicator world; - for(size_t nbi=0; nbi( interface[ nbi ] ); - barycenter_neighbours[ nbi ] = samurai::_cmpCellBarycenter( neighbourhood[ nbi ].mesh ); + logs << "# New load balancing" << std::endl; + + // build map of interval that needs to be sent. Warning, it does not work with classical std::map !! + auto flags = samurai::make_field("rank", mesh); + flags.fill( world.rank() ); + + // give access to rank & mesh of neighbour + std::vector & neighbourhood = mesh.mpi_neighbourhood(); - // surface or volume depending on dim - sv[ nbi ] = getSurfaceOrVolume( neighbourhood[ nbi ].mesh ); + std::size_t n_neighbours = neighbourhood.size(); + + std::vector loads; + double my_load = static_cast( samurai::cmptLoad( mesh ) ); + boost::mpi::all_gather( world, my_load, loads ); + + // Compute fluxes based on neighbourdhood graph + std::vector fluxes = samurai::cmptFluxes( mesh, 1 ); + + { + logs << "load : " << my_load << std::endl; + logs << "nneighbours : " << n_neighbours << std::endl; + logs << "neighbours : "; + for( size_t in=0; in + auto interface = samurai::_computeCartesianInterface( mesh ); - logs << s_ << std::endl; - } + // Invalidate fluxes with non adjacents neighbours, should not happen or might happen if load balancing exchange does not consider some + // direction or neighbourdhood is overkill (all meshes for example) + std::vector ncells_interface ( n_neighbours, 0 ); + for(size_t ni=0; ni("rank", mesh); - flags.fill( world.rank() ); + if( ncells_interface[ ni ] == 0 ) fluxes[ ni ] = 0; + } - constexpr auto fdist = samurai::Distance_t::L1; + // bary center of current mesh - no weight on cells + // Coord_t barycenter = _cmpIntervalBarycenter( mesh[ mesh_id_t::cells ] ); + Coord_t barycenter = _barycenterWithFlags( mesh, flags ); + logs << "Domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; - double currentSV = getSurfaceOrVolume( mesh ); + // std::vector barycenter_interface_neighbours( n_neighbours ); + std::vector barycenter_neighbours( n_neighbours ); - std::vector mload( static_cast( world.size() ), 0 ); - for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ + for(size_t nbi=0; nbi( mesh, flags, neighbourhood[ nbi ].rank, neighbourhood[ nbi ].mesh ); - auto cc = cell.center(); + // debug + auto s_ = fmt::format( "Barycenter neighbour # {}: ({}, {})", neighbourhood[ nbi ].rank, + barycenter_neighbours[ nbi ]( 0 ), + barycenter_neighbours[ nbi ]( 1 ) ); - // process that might get the interval - int winner_id = -1; + logs << s_ << std::endl; + } - // double winner_dist = std::numeric_limits::max(); - // double winner_dist = samurai::getDistance( cell, barycenter ) / loads[ world.rank() ]; - - // double coeff_current = currentSV / loads[ static_cast( world.rank() ) ]; - double coeff_current = 1.; // loads[ static_cast( world.rank() ) ]; - double winner_dist = samurai::getDistance( cc, barycenter ) * coeff_current; + constexpr auto fdist = samurai::Distance_t::L1; - // logs << fmt::format("\t\t> Cell : ({},{}), current mesh dist : {}", cc(0), cc(1), winner_dist ) << std::endl; + double currentSV = getSurfaceOrVolume( mesh ); - // select the neighbour - for( std::size_t ni=0; ni mload( static_cast( world.size() ), 0 ); + for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ - auto neighbour_rank = static_cast( neighbourhood[ ni ].rank ); + auto cc = cell.center(); - if( fluxes[ ni ] >= 0 ) continue; // skip neighbour that will recv + // process that might get the interval + int winner_id = -1; + double winner_dist = samurai::getDistance( cc, barycenter ) / currentSV; - // this might fix ilots but require neighbour update - // double dist = std::min( distance_inf( mid_point, barycenter_interface_neighbours[ ni ] ), - // distance_inf( mid_point, barycenter_neighbours[ ni ] ) ); + // logs << fmt::format("\t\t> Cell : ({},{}), current mesh dist : {}", cc(0), cc(1), winner_dist ) << std::endl; - // double dist = samurai::getDistance( cell, barycenter_interface_neighbours[ ni ] ) / loads[ neighbour_rank ]; - // double coeff = sv[ ni ] / ( loads[ neighbour_rank ] ) ; // / sv[ ni ]; - double coeff = 1.; // loads[ neighbour_rank ] ; // mload[ neighbour_rank ]; - double dist = samurai::getDistance( cell.center(), barycenter_neighbours[ ni ] ) * coeff; + // select the neighbour + for( std::size_t ni=0; ni Dist to neighbour {} : {}", neighbour_rank, dist ) << std::endl; + auto neighbour_rank = static_cast( neighbourhood[ ni ].rank ); - if( dist < winner_dist && ncells_interface[ ni ] > 0 ){ - winner_id = static_cast( ni ); - winner_dist = dist; + if( fluxes[ ni ] >= 0 ) continue; // skip neighbour that will recv + + double dist = samurai::getDistance( cell.center(), barycenter_neighbours[ ni ] ); + + // logs << fmt::format("\t\t\t> Dist to neighbour {} : {}", neighbour_rank, dist ) << std::endl; + + if( dist < winner_dist && ncells_interface[ ni ] > 0 ){ + winner_id = static_cast( ni ); + winner_dist = dist; + } + + } - mload[ neighbour_rank ] += 1; + if( winner_id >= 0 ){ + assert( winner_id < neighbourhood.size() ); + flags[ cell ] = neighbourhood[ static_cast( winner_id ) ].rank; - // if( mload[ neighbour_rank ] >= ( - fluxes[ ni ] ) ) fluxes[ ni ] = 0; + mload[ neighbourhood[ winner_id ].rank ] ++; } + + }); + + // update barycenter + barycenter = _barycenterWithFlags( mesh, flags ); + logs << "Updated domain barycenter : " << fmt::format( " barycenter : ({}, {})", barycenter(0), barycenter(1) ) << std::endl; + for(size_t nbi=0; nbi( mesh, flags, neighbourhood[ nbi ].rank, neighbourhood[ nbi ].mesh ); + // debug + auto s_ = fmt::format( "Updated Barycenter neighbour # {}: ({}, {})", neighbourhood[ nbi ].rank, + barycenter_neighbours[ nbi ]( 0 ), + barycenter_neighbours[ nbi ]( 1 ) ); + + logs << s_ << std::endl; } - // logs << fmt::format("\t\t\t> Cell given to process #{}", neighbourhood[ static_cast( winner_id ) ].rank ) << std::endl; + // second pass + for_each_cell( mesh[ Mesh_t::mesh_id_t::cells ], [&]( const auto& cell ){ + + auto cc = cell.center(); + + // process that might get the interval + int winner_id = -1; + double winner_dist = samurai::getDistance( cc, barycenter ) / currentSV; + + // logs << fmt::format("\t\t> Cell : ({},{}), current mesh dist : {}", cc(0), cc(1), winner_dist ) << std::endl; - if( winner_id >= 0 ){ - assert( winner_id < neighbourhood.size() ); - flags[ cell ] = neighbourhood[ static_cast( winner_id ) ].rank; + // select the neighbour + for( std::size_t ni=0; ni( neighbourhood[ ni ].rank ); + + if( fluxes[ ni ] + mload[ ni ] >= 0 ) continue; // skip neighbour that will recv + + double dist = samurai::getDistance( cell.center(), barycenter_neighbours[ ni ] ); + + // logs << fmt::format("\t\t\t> Dist to neighbour {} : {}", neighbour_rank, dist ) << std::endl; + + if( dist < winner_dist && ncells_interface[ ni ] > 0 ){ + winner_id = static_cast( ni ); + winner_dist = dist; + } + + } + + if( winner_id >= 0 ){ + assert( winner_id < neighbourhood.size() ); + flags[ cell ] = neighbourhood[ static_cast( winner_id ) ].rank; + + mload[ neighbourhood[ winner_id ].rank ] ++; + } + + }); + + for( std::size_t ni=0; ni Date: Thu, 25 Jul 2024 10:46:00 +0200 Subject: [PATCH 166/170] fix new name for LB --- demos/FiniteVolume/linear_convection.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index 7723698b1..ed4398e4c 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -161,7 +161,8 @@ int main(int argc, char* argv[]) auto conv = samurai::make_convection_weno5(velocity); // SFC_LoadBalancer_interval balancer; - Load_balancing::Life balancer; + // Load_balancing::Life balancer; + Load_balancing::GlobalCriteria balancer; // Void_LoadBalancer balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; @@ -204,10 +205,6 @@ int main(int argc, char* argv[]) balancer.load_balance(mesh, u); samurai::times::timers.stop("tloop.lb:"+balancer.getName()); - // std::string _stats = fmt::format("statistics_process_{}", world.rank()); - // samurai::Statistics s(_stats); - // s("statistics", mesh); - } // Move to next timestep From b4ebc2b0608394b861e5594f0b297217a7bf42bb Mon Sep 17 00:00:00 2001 From: sbstndb/sbstndbs Date: Tue, 25 Feb 2025 17:16:24 +0100 Subject: [PATCH 167/170] fix: Rebase working in single - Now the rebase works fine in single rank - Remove ponio - Remove some load_balancing and mpi codes in examples --- demos/FiniteVolume/CMakeLists.txt | 9 --------- demos/FiniteVolume/advection_2d.cpp | 11 +++++----- demos/FiniteVolume/linear_convection.cpp | 12 ++++++----- include/samurai/boundary.hpp | 2 +- include/samurai/load_balancing.hpp | 20 +++++++++++++------ include/samurai/load_balancing_diffusion.hpp | 6 ++++-- .../load_balancing_diffusion_interval.hpp | 4 +++- include/samurai/load_balancing_force.hpp | 4 +++- include/samurai/load_balancing_life.hpp | 5 +++-- include/samurai/load_balancing_sfc.hpp | 4 +++- include/samurai/load_balancing_sfc_w.hpp | 5 +++-- include/samurai/load_balancing_void.hpp | 4 +++- include/samurai/timers.hpp | 4 ---- 13 files changed, 50 insertions(+), 40 deletions(-) diff --git a/demos/FiniteVolume/CMakeLists.txt b/demos/FiniteVolume/CMakeLists.txt index 829c86d0e..a5f41a5b8 100644 --- a/demos/FiniteVolume/CMakeLists.txt +++ b/demos/FiniteVolume/CMakeLists.txt @@ -30,13 +30,6 @@ endif() add_executable(finite-volume-linear-convection linear_convection.cpp) target_link_libraries(finite-volume-linear-convection samurai CLI11::CLI11 ${MPI_LIBRARIES}) -# Test with Ponio (required CXX 20) -find_package(ponio CONFIG REQUIRED) - -add_executable(finite-volume-linear-convection-ponio linear_convection_ponio.cpp) -target_include_directories(finite-volume-linear-convection-ponio PUBLIC /Users/strafella/work/ponio/ponio/include/) -target_link_libraries(finite-volume-linear-convection-ponio samurai CLI11::CLI11 ${MPI_LIBRARIES} ponio::ponio ponio::project_options ponio::project_warnings) - add_executable(finite-volume-amr-burgers-hat AMR_Burgers_Hat.cpp) target_link_libraries(finite-volume-amr-burgers-hat samurai CLI11::CLI11) @@ -64,8 +57,6 @@ target_link_libraries(finite-volume-advection-2d-user-bc samurai CLI11::CLI11) add_executable(finite-volume-scalar-burgers-2d scalar_burgers_2d.cpp) target_link_libraries(finite-volume-scalar-burgers-2d samurai CLI11::CLI11) -add_executable(finite-volume-linear-convection linear_convection.cpp) -target_link_libraries(finite-volume-linear-convection samurai CLI11::CLI11) add_executable(finite-volume-burgers burgers.cpp) target_link_libraries(finite-volume-burgers samurai CLI11::CLI11) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index 51454a5e2..f2d268022 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -304,7 +304,7 @@ int main(int argc, char* argv[]) const int iof = 1; - SFC_LoadBalancer_interval balancer; +// SFC_LoadBalancer_interval ealancer; // Void_LoadBalancer balancer; // Diffusion_LoadBalancer_cell balancer; @@ -313,21 +313,22 @@ int main(int argc, char* argv[]) // Load_balancing::SFCw balancer; std::ofstream logs; +#ifdef SAMURAI_WITH_MPI boost::mpi::communicator world; logs.open( fmt::format("log_{}.dat", world.rank()), std::ofstream::app ); - +#endif while (t != Tf) { - bool reqBalance = balancer.require_balance( mesh ); +// bool reqBalance = balancer.require_balance( mesh ); - if( reqBalance ) std::cerr << "\t> Load Balancing required !!! " << std::endl; +// if( reqBalance ) std::cerr << "\t> Load Balancing required !!! " << std::endl; // if ( ( nt % nt_loadbalance == 0 || reqBalance ) && nt > 1 ) if ( ( nt % nt_loadbalance == 0 ) && nt > 1 ) // if ( reqBalance && nt > 1 ) { samurai::times::timers.start("load-balancing"); - balancer.load_balance(mesh, u); +// balancer.load_balance(mesh, u); samurai::times::timers.stop("load-balancing"); } diff --git a/demos/FiniteVolume/linear_convection.cpp b/demos/FiniteVolume/linear_convection.cpp index ed4398e4c..1c3b4e380 100644 --- a/demos/FiniteVolume/linear_convection.cpp +++ b/demos/FiniteVolume/linear_convection.cpp @@ -58,12 +58,12 @@ int main(int argc, char* argv[]) //--------------------// // Program parameters // //--------------------// +#ifdef SAMURAI_WITH_MPI boost::mpi::communicator world; - +#endif // Simulation parameters double left_box = -1; double right_box = 1; ->>>>>>> 7f2d425 (update linear convection with load balancing) // Time integration double Tf = 3; @@ -156,13 +156,14 @@ int main(int argc, char* argv[]) // Convection operator samurai::VelocityVector velocity; velocity.fill(1); + velocity(1) = -1 ; // origin weno5 auto conv = samurai::make_convection_weno5(velocity); // SFC_LoadBalancer_interval balancer; // Load_balancing::Life balancer; - Load_balancing::GlobalCriteria balancer; +// Load_balancing::GlobalCriteria balancer; // Void_LoadBalancer balancer; // Diffusion_LoadBalancer_cell balancer; // Diffusion_LoadBalancer_interval balancer; @@ -199,14 +200,15 @@ int main(int argc, char* argv[]) while (t != Tf) { - if (nt % nt_loadbalance == 0 && nt > 1 ) +/** + if (nt % nt_loadbalance == 0 && nt > 1 ) { samurai::times::timers.start("tloop.lb:"+balancer.getName()); balancer.load_balance(mesh, u); samurai::times::timers.stop("tloop.lb:"+balancer.getName()); } - + **/ // Move to next timestep t += dt; if (t > Tf) diff --git a/include/samurai/boundary.hpp b/include/samurai/boundary.hpp index e585ffe38..48531d56b 100644 --- a/include/samurai/boundary.hpp +++ b/include/samurai/boundary.hpp @@ -12,7 +12,7 @@ namespace samurai auto& cells = mesh[mesh_id_t::cells][level]; // auto& domain = mesh.domain(); // REBASE FIXME : I don't know if we need to override domain - auto& domain = mesh.subdomain(); + //auto& domain = mesh.subdomain(); auto max_level = domain.level(); // domain.level();//mesh[mesh_id_t::cells].max_level(); auto one_interval = layer_width << (max_level - level); diff --git a/include/samurai/load_balancing.hpp b/include/samurai/load_balancing.hpp index cbb3be14d..3c06ed3f5 100644 --- a/include/samurai/load_balancing.hpp +++ b/include/samurai/load_balancing.hpp @@ -6,18 +6,24 @@ #include #include -#include -#include -#include -#include -#include -#include +#include "algorithm.hpp" +#include "algorithm/utils.hpp" +#include "hilbert.hpp" +#include "mesh.hpp" +#include "morton.hpp" +#include "mr/mesh.hpp" // statistics #ifdef WITH_STATS #include #endif +#ifdef SAMURAI_WITH_MPI +#include +#endif + + +#ifdef SAMURAI_WITH_MPI namespace samurai { @@ -1589,6 +1595,8 @@ namespace samurai } } // namespace samurai +#endif + /** * This function perform a "fake" load-balancing by updating an integer scalar field containing the rank diff --git a/include/samurai/load_balancing_diffusion.hpp b/include/samurai/load_balancing_diffusion.hpp index 610cad769..29947c322 100644 --- a/include/samurai/load_balancing_diffusion.hpp +++ b/include/samurai/load_balancing_diffusion.hpp @@ -2,11 +2,12 @@ #include #include "load_balancing.hpp" -#include +#include "field.hpp" // for std::sort #include +#ifdef SAMURAI_WITH_MPI namespace Load_balancing{ class Diffusion : public samurai::LoadBalancer { @@ -393,4 +394,5 @@ namespace Load_balancing{ } }; -} \ No newline at end of file +} +#endif diff --git a/include/samurai/load_balancing_diffusion_interval.hpp b/include/samurai/load_balancing_diffusion_interval.hpp index 822eba13a..ca2a3acdb 100644 --- a/include/samurai/load_balancing_diffusion_interval.hpp +++ b/include/samurai/load_balancing_diffusion_interval.hpp @@ -3,6 +3,7 @@ #include #include "load_balancing.hpp" +#ifdef SAMURAI_WITH_MP template class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer> { @@ -454,4 +455,5 @@ class Diffusion_LoadBalancer_interval : public samurai::LoadBalancer #include "load_balancing.hpp" +#ifdef SAMURAI_WITH_MPI namespace Load_balancing{ class GlobalCriteria : public samurai::LoadBalancer { @@ -506,4 +507,5 @@ namespace Load_balancing{ }; -} \ No newline at end of file +} +#endif diff --git a/include/samurai/load_balancing_life.hpp b/include/samurai/load_balancing_life.hpp index 41f76a7f3..b9437c2e8 100644 --- a/include/samurai/load_balancing_life.hpp +++ b/include/samurai/load_balancing_life.hpp @@ -6,7 +6,7 @@ // for std::sort #include - +#ifdef SAMURAI_WITH_MPI namespace Load_balancing{ class Life : public samurai::LoadBalancer { @@ -80,4 +80,5 @@ namespace Load_balancing{ } }; -} \ No newline at end of file +} +#endif diff --git a/include/samurai/load_balancing_sfc.hpp b/include/samurai/load_balancing_sfc.hpp index 844d1e1ea..a1ed85b4f 100644 --- a/include/samurai/load_balancing_sfc.hpp +++ b/include/samurai/load_balancing_sfc.hpp @@ -4,6 +4,7 @@ #include "load_balancing.hpp" +#ifdef SAMURAI_WITH_MPI template class SFC_LoadBalancer_interval : public samurai::LoadBalancer> { @@ -279,4 +280,5 @@ class SFC_LoadBalancer_interval : public samurai::LoadBalancer @@ -344,4 +344,5 @@ namespace Load_balancing{ }; -} \ No newline at end of file +} +#endif diff --git a/include/samurai/load_balancing_void.hpp b/include/samurai/load_balancing_void.hpp index 65477f5b8..06f896bb8 100644 --- a/include/samurai/load_balancing_void.hpp +++ b/include/samurai/load_balancing_void.hpp @@ -8,6 +8,7 @@ #include #include "load_balancing.hpp" +#ifdef SAMURAI_WITH_MPI template class Void_LoadBalancer: public samurai::LoadBalancer> { @@ -47,4 +48,5 @@ class Void_LoadBalancer: public samurai::LoadBalancer> { return flags; } -}; \ No newline at end of file +}; +#endif diff --git a/include/samurai/timers.hpp b/include/samurai/timers.hpp index bdd4a889f..90b764e5a 100644 --- a/include/samurai/timers.hpp +++ b/include/samurai/timers.hpp @@ -29,10 +29,6 @@ namespace samurai #endif uint32_t ntimes; }; -struct Timer{ - double start, elapsed; - uint32_t ntimes; -}; class Timers { From 653109bc9574076efb734f1e2c82eefe2425e8cf Mon Sep 17 00:00:00 2001 From: sbstndb/sbstndbs Date: Wed, 26 Feb 2025 10:31:20 +0100 Subject: [PATCH 168/170] Remove useless MPI stuff - Remove useless MPI stuff from the git rebase @strafella --- include/samurai/samurai.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/samurai/samurai.hpp b/include/samurai/samurai.hpp index 4d13df6ac..bb04345af 100644 --- a/include/samurai/samurai.hpp +++ b/include/samurai/samurai.hpp @@ -31,12 +31,7 @@ namespace samurai #ifdef SAMURAI_WITH_MPI MPI_Init(&argc, &argv); - boost::mpi::communicator world; - - rank = world.rank(); - nproc = world.size(); - mpi_tag = "ON"; #endif times::timers.start("total runtime"); return app; From a43b6ed2e0126437b2d9b01754152dd22fb28a2b Mon Sep 17 00:00:00 2001 From: sbstndb/sbstndbs Date: Wed, 26 Feb 2025 10:32:10 +0100 Subject: [PATCH 169/170] Undo unwanted commit - Remove unwanted commit from git rebase @strafella --- include/samurai/mesh.hpp | 73 ++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/include/samurai/mesh.hpp b/include/samurai/mesh.hpp index 36e8050c2..207661b6b 100644 --- a/include/samurai/mesh.hpp +++ b/include/samurai/mesh.hpp @@ -775,6 +775,7 @@ namespace samurai */ /** delete for git rebase + **/ lcl_type subdomain_cells(start_level, m_domain.origin_point(), m_domain.scaling_factor()); auto subdomain_nb_intervals = m_domain.nb_intervals() / static_cast(size); auto subdomain_start = static_cast(rank) * subdomain_nb_intervals; @@ -793,47 +794,47 @@ namespace samurai this->m_cells[mesh_id_t::cells][start_level] = subdomain_cells; // end comment for git rebase - this->m_cells[mesh_id_t::cells][start_level] = {start_level, subdomain_box}; - **/ + // this->m_cells[mesh_id_t::cells][start_level] = {start_level, subdomain_box}; + - // m_mpi_neighbourhood.reserve(static_cast(size) - 1); - // for (int ir = 0; ir < size; ++ir) - // { - // if (ir != rank) - // { - // m_mpi_neighbourhood.push_back(ir); - // } - // } - - // // Neighbours - m_mpi_neighbourhood.reserve(static_cast(pow(3, dim) - 1)); - auto neighbour = [&](xt::xtensor_fixed> shift) + m_mpi_neighbourhood.reserve(static_cast(size) - 1); + for (int ir = 0; ir < size; ++ir) { - auto neighbour_rank = rank; - int product_of_preceding_sizes = 1; - for (std::size_t d = 0; d < dim; ++d) + if (ir != rank) { - neighbour_rank += product_of_preceding_sizes * shift[d]; - product_of_preceding_sizes *= sizes[d]; + m_mpi_neighbourhood.push_back(ir); } - return neighbour_rank; - }; + } - static_nested_loop( - [&](auto& shift) - { - if (xt::any(shift)) - { - for (std::size_t d = 0; d < dim; ++d) - { - if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) - { - return; - } - } - m_mpi_neighbourhood.push_back(neighbour(shift)); - } - }); + // // Neighbours + // m_mpi_neighbourhood.reserve(static_cast(pow(3, dim) - 1)); + // auto neighbour = [&](xt::xtensor_fixed> shift) + // { + // auto neighbour_rank = rank; + // int product_of_preceding_sizes = 1; + // for (std::size_t d = 0; d < dim; ++d) + // { + // neighbour_rank += product_of_preceding_sizes * shift[d]; + // product_of_preceding_sizes *= sizes[d]; + // } + // return neighbour_rank; + //}; + + //static_nested_loop( + // [&](auto& shift) + // { + // if (xt::any(shift)) + // { + // for (std::size_t d = 0; d < dim; ++d) + // { + // if (coords[d] + shift[d] < 0 || coords[d] + shift[d] >= sizes[d]) + // { + // return; + // } + // } + // m_mpi_neighbourhood.push_back(neighbour(shift)); + // } + // }); #endif } From d328e2be14cb33de2bc54e384398416ba5623a6c Mon Sep 17 00:00:00 2001 From: sbstndb/sbstndbs Date: Wed, 26 Feb 2025 14:08:39 +0100 Subject: [PATCH 170/170] Remove Stuff - Remove the specific configuration of the benchmark by @strafella - Now the file is similar to the master one --- demos/FiniteVolume/advection_2d.cpp | 97 +++++++++-------------------- 1 file changed, 29 insertions(+), 68 deletions(-) diff --git a/demos/FiniteVolume/advection_2d.cpp b/demos/FiniteVolume/advection_2d.cpp index f2d268022..76fa6a5f3 100644 --- a/demos/FiniteVolume/advection_2d.cpp +++ b/demos/FiniteVolume/advection_2d.cpp @@ -29,7 +29,7 @@ namespace fs = std::filesystem; template -auto init(Mesh& mesh, const double radius, const double x_center, const double y_center) +auto init(Mesh& mesh) { auto u = samurai::make_field("u", mesh); @@ -40,47 +40,17 @@ auto init(Mesh& mesh, const double radius, const double x_center, const double y [&](auto& cell) { auto center = cell.center(); - // const double radius = .2; - // const double x_center = 0.3; - // const double y_center = 0.3; + const double radius = .2; + const double x_center = 0.3; + const double y_center = 0.3; if (((center[0] - x_center) * (center[0] - x_center) + (center[1] - y_center) * (center[1] - y_center)) <= radius * radius) { - u[cell] += 1; + u[cell] = 1; } - - }); - - return u; -} - -template -auto init(Mesh& mesh, const std::vector & radius, const std::vector & rcenters ) -{ - auto u = samurai::make_field("u", mesh); - - u.fill( 0. ); - - samurai::for_each_cell( - mesh, - [&](auto& cell) - { - auto center = cell.center(); - - for(std::size_t nc=0; nc> min_corner = {0., 0.}; - xt::xtensor_fixed> max_corner = {4., 4.}; + xt::xtensor_fixed> max_corner = {1., 1.}; std::array a{ {1, 1} }; - double Tf = .9; + double Tf = .1; double cfl = 0.5; // Multiresolution parameters std::size_t min_level = 4; - std::size_t max_level = 6; + std::size_t max_level = 10; double mr_epsilon = 2.e-4; // Threshold used by multiresolution double mr_regularity = 1.; // Regularity guess for multiresolution bool correction = false; @@ -241,8 +211,8 @@ int main(int argc, char* argv[]) // Output parameters fs::path path = fs::current_path(); std::string filename = "FV_advection_2d"; - std::size_t nfiles = 10; - std::size_t nt_loadbalance = 10; // nombre d'iteratio entre les equilibrages + std::size_t nfiles = 1; + std::size_t nt_loadbalance = 1; // nombre d'iteration entre les equilibrages app.add_option("--min-corner", min_corner, "The min corner of the box")->capture_default_str()->group("Simulation parameters"); app.add_option("--max-corner", max_corner, "The max corner of the box")->capture_default_str()->group("Simulation parameters"); @@ -276,17 +246,8 @@ int main(int argc, char* argv[]) const double dt_save = Tf / static_cast(nfiles); double t = 0.; - std::vector rcenters, radii; - rcenters.emplace_back( x_center ); - rcenters.emplace_back( y_center ); - radii.emplace_back( radius ); - - rcenters = { 0.8, 0.8, - 2., 2., - 3., 1. }; - radii = { 0.8, 0.4, 0.2 }; + auto u = init(mesh); - auto u = init(mesh, radii, rcenters); samurai::make_bc>(u, 0.); auto unp1 = samurai::make_field("unp1", mesh); @@ -301,15 +262,17 @@ int main(int argc, char* argv[]) std::size_t nsave = 1; std::size_t nt = 0; - - const int iof = 1; -// SFC_LoadBalancer_interval ealancer; - // Void_LoadBalancer balancer; - // Diffusion_LoadBalancer_cell balancer; - // Diffusion_LoadBalancer_interval balancer; - // Load_balancing::Diffusion balancer; +// For now, void_balancer is verified and works properly +// Diffusion_LoadBalancer_cell not exist ??? +// Load_balancing::Diffusion donne de très mauvais resultats, peut-etre des parametres internes ? + + SFC_LoadBalancer_interval balancer; +// Void_LoadBalancer balancer; +// Diffusion_LoadBalancer_cell balancer; +// Diffusion_LoadBalancer_interval balancer; +// Load_balancing::Diffusion balancer; // Load_balancing::SFCw balancer; std::ofstream logs; @@ -319,16 +282,16 @@ int main(int argc, char* argv[]) #endif while (t != Tf) { -// bool reqBalance = balancer.require_balance( mesh ); + bool reqBalance = balancer.require_balance( mesh ); -// if( reqBalance ) std::cerr << "\t> Load Balancing required !!! " << std::endl; + if( reqBalance ) std::cerr << "\t> Load Balancing required !!! " << std::endl; // if ( ( nt % nt_loadbalance == 0 || reqBalance ) && nt > 1 ) if ( ( nt % nt_loadbalance == 0 ) && nt > 1 ) // if ( reqBalance && nt > 1 ) { samurai::times::timers.start("load-balancing"); -// balancer.load_balance(mesh, u); + balancer.load_balance(mesh, u); samurai::times::timers.stop("load-balancing"); } @@ -355,18 +318,16 @@ int main(int argc, char* argv[]) if (correction) { - // flux_correction(dt, a, u, unp1); + flux_correction(dt, a, u, unp1); } std::swap(u.array(), unp1.array()); - samurai::times::timers.start("I/O"); - if (t >= static_cast(nsave + 1) * dt_save || t == Tf || nt % iof == 0 ) + if (t >= static_cast(nsave + 1) * dt_save || t == Tf) { const std::string suffix = (nfiles != 1) ? fmt::format("_ite_{}", nsave++) : ""; save(path, filename, u, suffix); } - samurai::times::timers.stop("I/O"); }