From 4c27b0fbde96543407536e1278b3590cdcb372df Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 30 May 2025 19:38:05 +0200 Subject: [PATCH 01/94] Implemented shared memory parallelization for shot range loop --- CMakeLists.txt | 3 +- src/core/cabana_data.hpp | 54 +++ src/core/cell_system/CellStructure.cpp | 34 ++ src/core/cell_system/CellStructure.hpp | 58 +++ src/core/communication.cpp | 2 + src/core/communication.hpp | 8 + src/core/custom_verlet_list.hpp | 143 +++++++ src/core/forces.cpp | 48 ++- src/core/forces.hpp | 1 + src/core/forces_inline.hpp | 12 +- src/core/npt.cpp | 7 + src/core/short_range_cabana.cpp | 516 +++++++++++++++++++++++ src/core/system/System.cpp | 4 + src/core/system/System.hpp | 3 + testsuite/python/caliper.py | 1 + testsuite/python/integrator_npt_stats.py | 4 +- testsuite/python/unittest_decorators.py | 9 + 17 files changed, 899 insertions(+), 8 deletions(-) create mode 100644 src/core/cabana_data.hpp create mode 100644 src/core/custom_verlet_list.hpp create mode 100644 src/core/short_range_cabana.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 16ccc56d11a..c21412c5401 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -538,7 +538,8 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) FetchContent_Declare( kokkos GIT_REPOSITORY https://github.com/kokkos/kokkos.git - GIT_TAG 18b830e # version 4.6.1 with patches + GIT_TAG aba6e3caf2b8814fe6764a5d27b7b181253df14f # version 4.5.1 + #GIT_TAG 18b830e # version 4.6.1 with patches OVERRIDE_FIND_PACKAGE ) # cmake-format: on diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp new file mode 100644 index 00000000000..8f538fbff39 --- /dev/null +++ b/src/core/cabana_data.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2010-2025 The ESPResSo project + * + * This file is part of ESPResSo. + * + * ESPResSo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ESPResSo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#ifdef SHARED_MEMORY_PARALLELISM + +#include +#include "custom_verlet_list.hpp" +#include +#include + +using data_types = Cabana::MemberTypes; +using memory_space = Kokkos::SharedSpace; +using execution_space = Kokkos::DefaultExecutionSpace; + +using ListAlgorithm = Cabana::HalfNeighborTag; +using ListType = Cabana::CustomVerletList; + +class CabanaData { + ListType verlet_list; + std::unordered_map id_to_index; + std::vector index_to_id; + +public: + CabanaData() = default; + CabanaData(ListType verlet_list, std::unordered_map id_to_index) + : verlet_list(verlet_list), id_to_index(id_to_index) {} + CabanaData(ListType verlet_list, std::unordered_map id_to_index, std::vector index_to_id) + : verlet_list(verlet_list), id_to_index(id_to_index), index_to_id(index_to_id) {} + + ListType get_verlet_list() const { return verlet_list; } + std::unordered_map get_id_to_index() const { return id_to_index; } + std::vector get_index_to_id() const { return index_to_id; } + + ~CabanaData() {}; +}; +#endif diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index e803bafe648..d0af7e7e8c8 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -52,9 +52,42 @@ #include #ifdef SHARED_MEMORY_PARALLELISM +#include +#include "custom_verlet_list.hpp" +#include "cabana_data.hpp" #include #endif +#ifdef SHARED_MEMORY_PARALLELISM + +using data_types = Cabana::MemberTypes; +using memory_space = Kokkos::SharedSpace; +using execution_space = Kokkos::DefaultExecutionSpace; + +using ListAlgorithm = Cabana::HalfNeighborTag; +using ListType = Cabana::CustomVerletList; + + +CellStructure::~CellStructure() { + m_cabana_data.reset(); +} + +void CellStructure::set_cabana_data(std::unique_ptr data) { + m_cabana_data = std::move(data); +} + +CabanaData& CellStructure::get_cabana_data() { + return *m_cabana_data; +} + +void CellStructure::reset_cabana_data() { + m_rebuild_verlet_list = true; + m_cabana_data.reset(); +} + +#endif + + CellStructure::CellStructure(BoxGeometry const &box) : m_decomposition{std::make_unique(box)} {} @@ -234,6 +267,7 @@ void CellStructure::resort_particles(bool global_flag) { auto const &lebc = get_system().box_geo->lees_edwards_bc(); m_rebuild_verlet_list = true; + m_rebuild_cabana_verlet_list = true; m_le_pos_offset_at_last_resort = lebc.pos_offset; #ifdef ADDITIONAL_CHECKS diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 7c4cc1a2584..249af45f8e9 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -54,6 +54,17 @@ #include #include #include +#include + +// forward declaration to not have to import cabana +#ifdef SHARED_MEMORY_PARALLELISM +class CabanaData; +#endif + +template +concept ParticleCallback = requires(Callable c, Particle &p) { + { c(p) } -> std::same_as; +}; using ParticleUnaryOp = std::function; @@ -147,6 +158,7 @@ struct CellStructure : public System::Leaf { */ unsigned m_resort_particles = Cells::RESORT_NONE; bool m_rebuild_verlet_list = true; + bool m_rebuild_cabana_verlet_list = true; std::vector> m_verlet_list; double m_le_pos_offset_at_last_resort = 0.; /** @brief Verlet list skin. */ @@ -278,6 +290,7 @@ struct CellStructure : public System::Leaf { ParticleRange ghost_particles() const { return Cells::particles(decomposition().ghost_cells()); } + /** @brief whether to use parallel version of @ref for_each_local_particle */ bool use_parallel_for_each_local_particle() const { #ifdef SHARED_MEMORY_PARALLELISM @@ -646,6 +659,51 @@ struct CellStructure : public System::Leaf { } } +#ifdef SHARED_MEMORY_PARALLELISM +private: + std::unique_ptr m_cabana_data; + +public: + void set_cabana_data(std::unique_ptr data); + CabanaData& get_cabana_data(); + void reset_cabana_data(); + + virtual ~CellStructure(); + + bool get_rebuild_verlet_list() const { return m_rebuild_verlet_list; } + bool get_rebuild_cabana_verlet_list() const { return m_rebuild_cabana_verlet_list; } + + template + void cabana_link_cell(Kernel kernel) { + auto const local_cells_span = decomposition().local_cells(); + auto const first = boost::make_indirect_iterator(local_cells_span.begin()); + auto const last = boost::make_indirect_iterator(local_cells_span.end()); + + Algorithm::link_cell(first, last, [&kernel](Particle &p1, Particle &p2) { + kernel(p1, p2); + }); + } + + template + void cabana_verlet_list_loop(Kernel kernel, + const VerletCriterion &verlet_criterion) { + if (m_rebuild_cabana_verlet_list) { + m_verlet_list.clear(); + + link_cell([&](Particle &p1, Particle &p2, Distance const &d) { + if (verlet_criterion(p1, p2, d)) { + m_verlet_list.emplace_back(&p1, &p2); + } + }); + m_rebuild_cabana_verlet_list = false; + } + for (auto const &pair : m_verlet_list) { + kernel(*pair.first, *pair.second); + } + } +#endif + +private: /** Non-bonded pair loop with verlet lists. * * @param pair_kernel Kernel to apply diff --git a/src/core/communication.cpp b/src/core/communication.cpp index 38b5ed84091..a6f56c37741 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -34,6 +34,8 @@ #ifdef SHARED_MEMORY_PARALLELISM #include #include +#include "system/System.hpp" +#include "cell_system/CellStructure.hpp" #endif #include diff --git a/src/core/communication.hpp b/src/core/communication.hpp index c5520935993..dfeee2a5f0a 100644 --- a/src/core/communication.hpp +++ b/src/core/communication.hpp @@ -131,4 +131,12 @@ void init(std::shared_ptr mpi_env); void deinit(); } // namespace Communication +struct MpiContainerUnitTest { + std::shared_ptr m_mpi_env; + MpiContainerUnitTest(int argc, char **argv) { + m_mpi_env = mpi_init(argc, argv); + Communication::init(m_mpi_env); + } + ~MpiContainerUnitTest() { Communication::deinit(); } +}; #endif diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp new file mode 100644 index 00000000000..8a990104ab5 --- /dev/null +++ b/src/core/custom_verlet_list.hpp @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2010-2022 The ESPResSo project + * + * This file is part of ESPResSo. + * + * ESPResSo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ESPResSo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once + +#ifdef SHARED_MEMORY_PARALLELISM + +#include + +namespace Cabana +{ +// ONLY FOR 2D LAYOUT, OTHERWISE NEIGHBOR LIST INTERFACE IMPLEMENTATION WILL CAUSE PROBLEMS (NOT IMPLEMENTED) +template +class CustomVerletList : public VerletList +{ + public: + using Base = VerletList; + + // Default constructor + CustomVerletList() : Base() {} + + // Custom constructor + template + CustomVerletList(PositionSlice x, const std::size_t begin, const std::size_t end, + const std::size_t max_neigh) + { + initializeData(x.size(), max_neigh); + } + virtual ~CustomVerletList() {}; + + +public: + Kokkos::View counts; + Kokkos::View neighbors; + + // Method to initialize _data without filling neighbors + KOKKOS_INLINE_FUNCTION + void initializeData(const std::size_t num_particles, const std::size_t max_neigh) + { + counts = Kokkos::View("num_neighbors", num_particles); + neighbors = Kokkos::View( + Kokkos::ViewAllocateWithoutInitializing("neighbors"), + num_particles, max_neigh); + } + + // Method to dynamically expand the size of max_neighbors + KOKKOS_INLINE_FUNCTION + void expandMaxNeighbors(const std::size_t new_max_neigh) + { + // Create a new view with the larger size + Kokkos::View new_neighbors( + Kokkos::ViewAllocateWithoutInitializing("neighbors"), + neighbors.extent(0), new_max_neigh); + + // Copy existing data to the new view + Kokkos::parallel_for("copy_neighbors", neighbors.extent(0), KOKKOS_LAMBDA(const int i) { + for (std::size_t j = 0; j < counts(i); ++j) { + new_neighbors(i, j) = neighbors(i, j); + } + }); + + // Replace the old view with the new view + neighbors = new_neighbors; + } + + // Method to add a neighbor + KOKKOS_INLINE_FUNCTION + void addNeighbor(const int pid, const int nid) + { + std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); + if (count >= neighbors.extent(1)) { + expandMaxNeighbors(neighbors.extent(1) * 2); + } + neighbors(pid, count) = nid; + } +}; + +template +class NeighborList< + CustomVerletList> +{ + public: + //! Kokkos memory space. + using memory_space = MemorySpace; + //! Neighbor list type. + using list_type = + CustomVerletList; + + //! Get the total number of neighbors across all particles. + KOKKOS_INLINE_FUNCTION + static std::size_t totalNeighbor( const list_type& list ) + { + std::size_t num_p = list._data.counts.size(); + for ( std::size_t i = 0; i < num_p; ++i ) + num_p += list.counts( i ); + return num_p; + } + + //! Get the maximum number of neighbors per particle. + KOKKOS_INLINE_FUNCTION + static std::size_t maxNeighbor( const list_type& list ) + { + // Stored during neighbor search. + return list.max_n; + } + + //! Get the number of neighbors for a given particle index. + KOKKOS_INLINE_FUNCTION + static std::size_t numNeighbor( const list_type& list, + const std::size_t particle_index ) + { + return list.counts( particle_index ); + } + + //! Get the id for a neighbor for a given particle index and the index of + //! the neighbor relative to the particle. + KOKKOS_INLINE_FUNCTION + static std::size_t getNeighbor( const list_type& list, + const std::size_t particle_index, + const std::size_t count) + { + return list.neighbors( particle_index, count ); + } +}; + +} + +#endif diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 43d0ca27e6b..9908e7320c9 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -59,6 +59,11 @@ #include #endif +#ifdef SHARED_MEMORY_PARALLELISM +#include +#include "short_range_cabana.cpp" +#endif + #include #include #include @@ -177,13 +182,15 @@ void System::System::calculate_forces() { auto const collision_detection_cutoff = INACTIVE_CUTOFF; #endif - short_range_loop( + // interaction kernel is defined + auto bond_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), &bonded_ias = *bonded_ias, &bond_breakage = *bond_breakage, &box_geo = *box_geo]( Particle &p1, int bond_id, std::span partners) { return add_bonded_force(p1, bond_id, partners, bonded_ias, bond_breakage, box_geo, coulomb_kernel_ptr); - }, + }; + auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), dipoles_kernel_ptr = get_ptr(dipoles_kernel), elc_kernel_ptr = get_ptr(elc_kernel), @@ -205,12 +212,42 @@ void System::System::calculate_forces() { collision_detection.detect_collision(p1, p2, d.dist2); } #endif - }, + }; + +#ifdef SHARED_MEMORY_PARALLELISM + auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); + auto dipoles_kernel_ptr= get_ptr(dipoles_kernel); + auto elc_kernel_ptr = get_ptr(elc_kernel); + auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); + cabana_short_range( + bond_kernel, + *bonded_ias, + coulomb_kernel_ptr, dipoles_kernel_ptr, elc_kernel_ptr, coulomb_u_kernel_ptr, +#ifdef COLLISION_DETECTION + collision_detection, +#endif + *cell_structure, + maximal_cutoff(), + bonded_ias->maximal_cutoff(), + *thermostat, + *box_geo, + *nonbonded_ias, + particles, + cell_structure->ghost_particles(), + VerletCriterion<>{*this, cell_structure->get_verlet_skin(), + get_interaction_range(), coulomb_cutoff, dipole_cutoff, + collision_detection_cutoff} + ); +#else + short_range_loop( + bond_kernel, + pair_kernel, *cell_structure, maximal_cutoff(), bonded_ias->maximal_cutoff(), VerletCriterion<>{*this, cell_structure->get_verlet_skin(), get_interaction_range(), coulomb_cutoff, dipole_cutoff, collision_detection_cutoff}); - + +#endif constraints->add_forces(particles, get_sim_time()); oif_global->calculate_forces(); @@ -269,6 +306,9 @@ void calc_long_range_forces(const ParticleRange &particles) { } #ifdef NPT +void npt_add_virial_force_contribution(const Utils::Vector3d &virial) { + ::System::get_system().npt_add_virial_contribution(virial); +} void npt_add_virial_force_contribution(const Utils::Vector3d &force, const Utils::Vector3d &d) { ::System::get_system().npt_add_virial_contribution(force, d); diff --git a/src/core/forces.hpp b/src/core/forces.hpp index e0b2f139245..f3be82654e4 100644 --- a/src/core/forces.hpp +++ b/src/core/forces.hpp @@ -42,6 +42,7 @@ void calc_long_range_forces(ParticleRange const &particles); #ifdef NPT /** Update the NpT virial */ +void npt_add_virial_force_contribution(Utils::Vector3d const &virial); void npt_add_virial_force_contribution(Utils::Vector3d const &force, Utils::Vector3d const &d); void npt_add_virial_diagonalSum_contribution(double diagonal_sum); diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 9d1e00159bd..00ac00c491b 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -170,6 +170,12 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, return out; } + +#ifdef SHARED_MEMORY_PARALLELISM +using ReturnType = ParticleForce; +#else +using ReturnType = void; +#endif /** Calculate non-bonded forces between a pair of particles and update their * forces and torques. * @param[in,out] p1 particle 1. @@ -186,7 +192,7 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, * @param[in] elc_kernel ELC force correction kernel. * @param[in] coulomb_u_kernel Coulomb energy kernel. */ -inline void add_non_bonded_pair_force( +inline ReturnType add_non_bonded_pair_force( Particle &p1, Particle &p2, Utils::Vector3d const &d, double dist, double dist2, IA_parameters const &ia_params, Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, @@ -275,8 +281,12 @@ inline void add_non_bonded_pair_force( /* add total non-bonded forces to particles */ /***********************************************/ +#ifdef SHARED_MEMORY_PARALLELISM + return pf; +#else p1.force_and_torque() += pf; p2.force_and_torque() += calc_opposing_force(pf, d); +#endif } /** Compute the bonded interaction force between particle pairs. diff --git a/src/core/npt.cpp b/src/core/npt.cpp index 6a554671a87..623c4fb1a48 100644 --- a/src/core/npt.cpp +++ b/src/core/npt.cpp @@ -148,4 +148,11 @@ void System::System::npt_add_virial_contribution(Utils::Vector3d const &force, npt_inst_pressure->p_vir += hadamard_product(force, d); } } + +void System::System::npt_add_virial_contribution(Utils::Vector3d const &virial) { + if ((propagation->integ_switch == INTEG_METHOD_NPT_ISO_AND) or + (propagation->integ_switch == INTEG_METHOD_NPT_ISO_MTK)) { + npt_inst_pressure->p_vir += virial; + } +} #endif // NPT diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp new file mode 100644 index 00000000000..5997264d75a --- /dev/null +++ b/src/core/short_range_cabana.cpp @@ -0,0 +1,516 @@ +/* + * Copyright (C) 2010-2025 The ESPResSo project + * + * This file is part of ESPResSo. + * + * ESPResSo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ESPResSo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "config/config.hpp" + +#include "cell_system/CellStructure.hpp" + +#ifdef CALIPER +#include +#endif + +#ifdef SHARED_MEMORY_PARALLELISM + +#include +#include "cabana_data.hpp" +#include "custom_verlet_list.hpp" +#include +#include +#include +#include + + + +template +inline void write_particle(Particle const &p, std::unordered_map const &id_to_index, SliceDouble3 &s_position, SliceDouble3 &s_force, SliceDouble3 &s_torque, SliceInt &s_id, SliceInt &s_type) { + auto const pos = p.pos(); + auto const id = id_to_index.at(p.id()); + s_position(id, 0) = pos[0]; + s_position(id, 1) = pos[1]; + s_position(id, 2) = pos[2]; + s_id(id) = p.id(); + s_type(id) = p.type(); + s_force(id, 0) = 0.0; + s_force(id, 1) = 0.0; + s_force(id, 2) = 0.0; + s_torque(id, 0) = 0.0; + s_torque(id, 1) = 0.0; + s_torque(id, 2) = 0.0; +} + +template +void cabana_short_range(BondKernel bond_kernel, + [[maybe_unused]] BondedInteractionsMap const &bonded_ias, + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel, +#ifdef COLLISION_DETECTION + std::shared_ptr collision_detection, +#endif + CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, + Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, + InteractionsNonBonded &nonbonded_ias, + ParticleRange particles, ParticleRange ghost_particles, + VerletCriterion const &verlet_criterion = {}) { +#ifdef CALIPER + CALI_CXX_MARK_FUNCTION; +#endif + + #ifdef CALIPER + CALI_MARK_BEGIN("Espresso - Bond Kernel"); + #endif + + assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); + + if (bond_cutoff >= 0.) { + cell_structure.bond_loop(bond_kernel); + } + + #ifdef CALIPER + CALI_MARK_END("Espresso - Bond Kernel"); + #endif + + // Cabana short range loop + if (pair_cutoff > 0.) { + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + // =================================================== + // Setup Cabana Variables + // =================================================== +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Setup"); +#endif + // Dont know where to do this better + using data_types = Cabana::MemberTypes; + using memory_space = Kokkos::SharedSpace; + using execution_space = Kokkos::DefaultExecutionSpace; + + using ListAlgorithm = Cabana::HalfNeighborTag; + using ListType = Cabana::CustomVerletList; + + //Number of threads + const int num_threads = execution_space().concurrency(); + + const int vector_length = 8; +#ifdef CALIPER + CALI_MARK_END("Cabana - Setup"); +#endif + + // =================================================== + // Count unique particles and create Index map + // =================================================== +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Index map"); +#endif + std::unordered_map id_to_index{}; + std::vector index_to_id{}; + int index = 0; + + bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); + + CabanaData saved_data; + + // Load saved data if we do not have to rebuild + if (!rebuild) { + saved_data = cell_structure.get_cabana_data(); + } + + // If we have to rebuild, we need to count the particles and create a new map + if (rebuild) { + + for (auto const& p : particles) { + id_to_index[p.id()] = index; + index_to_id.emplace_back(p.id()); + index++; + } + + for (auto const& p : ghost_particles) { + if (not id_to_index.contains(p.id())) { + id_to_index[p.id()] = index; + index_to_id.emplace_back(p.id()); + index++; + } + } + } else { + // If we do not rebuild we can use the saved map + id_to_index = saved_data.get_id_to_index(); + index_to_id = saved_data.get_index_to_id(); + index = id_to_index.size(); + } + + const int number_of_unique_particles = index; +#ifdef CALIPER + CALI_MARK_END("Cabana - Index map"); +#endif + + // =================================================== + // Create and fill particle storage + // =================================================== +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Fill particle storage"); +#endif + Cabana::AoSoA particle_storage("particles", number_of_unique_particles); + auto slice_position = Cabana::slice<0>(particle_storage); + auto slice_force = Cabana::slice<1>(particle_storage); + auto slice_torque = Cabana::slice<2>(particle_storage); + auto slice_id = Cabana::slice<3>(particle_storage); + auto slice_type = Cabana::slice<4>(particle_storage); + for (auto const& p : particles) { + write_particle(p, id_to_index, slice_position, slice_force, slice_torque, slice_id, slice_type); + } + using TP = decltype(slice_position); + using TF = decltype(slice_force); + using TR = decltype(slice_torque); + using TT = decltype(slice_type); + + Kokkos::View virial_all("virial_all"); + Kokkos::View force_local_thread("force_local_thread", number_of_unique_particles, 3, num_threads); + + for (auto const& p : ghost_particles) { + // if the ghost is not in the previous map, but mpi moved it to this rank? + // it will not have neighbors because we did not rebuild the verlet list. + if (not id_to_index.contains(p.id())) { + continue; + } + write_particle(p, id_to_index, slice_position, slice_force, slice_torque, slice_id, slice_type); + } +#ifdef CALIPER + CALI_MARK_END("Cabana - Fill particle storage"); +#endif + + // =================================================== + // Get Verlet Pairs and Fill list + // =================================================== +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Verlet List"); +#endif + ListType verlet_list; + + // Rebuild verlet list if needed + if (rebuild) { + + verlet_list = ListType(slice_position, 0, slice_position.size(), 64); + + auto kernel = [&](Particle const &p1, Particle const &p2) { + verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); + }; + + cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); + } else { + // Else use the saved verlet list + verlet_list = saved_data.get_verlet_list(); + } + + // Save data for next iteration if we just rebuilt + if (rebuild) { + CabanaData new_data(verlet_list, id_to_index, index_to_id); + cell_structure.set_cabana_data(std::make_unique(new_data)); + } + + // fill customverletlist with pairs + struct FirstNeighborKernel { + const CellStructure* cell; + [[maybe_unused]] const BondedInteractionsMap &bonded_ias; + const InteractionsNonBonded &nonbonded_ias; + const Thermostat::Thermostat &thermostat; + const BoxGeometry &box_geo; + std::vector &index_to_id; + TP &slice_position; + TF &slice_force; + Kokkos::View force_local_thread; + TR &slice_torque; + TT &slice_type; +#ifdef COLLISION_DETECTION + //std::shared_ptr collision_detection; + mutable CollisionDetection::CollisionDetection collision_detection; +#endif + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; + + Kokkos::View virial_all; + //Kokkos::View virial_all; + + int num_threads; + int mpi_rank; + + FirstNeighborKernel(const CellStructure* cell_, + [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, + const InteractionsNonBonded &nonbonded_ias_, + const Thermostat::Thermostat &thermostat_, + const BoxGeometry &box_geo_, + std::vector &index_to_id_, + TP &slice_position_, + TF &slice_force_, + Kokkos::View &force_local_thread_, + TR &slice_torque_, + TT &slice_type_, +#ifdef COLLISION_DETECTION + //std::shared_ptr collision_detection_, + CollisionDetection::CollisionDetection collision_detection_, +#endif + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, + Kokkos::View virial_all_, + //Kokkos::View virial_all_, + int num_threads_, + int mpi_rank_ + ) + : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + thermostat(thermostat_), box_geo(box_geo_), index_to_id(index_to_id_), + slice_position(slice_position_), slice_force(slice_force_), force_local_thread(force_local_thread_), + slice_torque(slice_torque_), slice_type(slice_type_), + collision_detection(collision_detection_), + coulomb_kernel(coulomb_kernel_), + dipoles_kernel(dipoles_kernel_), + elc_kernel(elc_kernel_), + coulomb_u_kernel(coulomb_u_kernel_), + virial_all(virial_all_), + num_threads(num_threads_), + mpi_rank(mpi_rank_) + {} + + KOKKOS_INLINE_FUNCTION + void operator()(int i, int j) const { + Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), slice_position(i, 2)}; + Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), slice_position(j, 2)}; + + Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); + auto const dist = d.norm(); + auto const dist2 = dist * dist; + + auto p1 = cell->get_local_particle(index_to_id.at(i)); + auto p2 = cell->get_local_particle(index_to_id.at(j)); + if (p1 == nullptr or p2 == nullptr) return; + //auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); + //std::cout << thread_id << " " << index_to_id.size() << " Find " << p1 << " " << p2 << "\n"; + //std::cout << index_to_id.size() << " pos_i " << p1->pos() << "\n"; + //std::cout << index_to_id.size() << " pos_j " << p2->pos() << "\n"; + //if (dist > pair_cutoff) { + // return; + //} + + IA_parameters const& ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); + + ParticleForce pf{}; + + /***********************************************/ + /* non-bonded pair potentials */ + /***********************************************/ + + if (dist < ia_params.max_cut) { +#ifdef EXCLUSIONS + if (do_nonbonded(*p1, *p2)) { +#endif + pf += calc_central_radial_force(ia_params, d, dist); +#ifdef THOLE + pf.f += thole_pair_force(*p1, *p2, ia_params, d, dist, bonded_ias, + coulomb_kernel); +#endif + pf += calc_non_central_force(*p1, *p2, ia_params, d, dist); +#ifdef EXCLUSIONS + } +#endif + } + +#ifdef NPT + //npt_add_virial_force_contribution(pf.f, d); + auto virial = hadamard_product(pf.f, d); + //auto virial = std::accumulate(virial_vec.begin(), virial_vec.end(), 0.0); +#endif + +#ifdef ELECTROSTATICS + // real-space electrostatic charge-charge interaction + auto const q1q2 = p1->q() * p2->q(); + if (q1q2 != 0. and coulomb_kernel != nullptr) { + pf.f += (*coulomb_kernel)(q1q2, d, dist); +#ifdef NPT + //npt_add_virial_diagonalSum_contribution( + // (*coulomb_u_kernel)(*p1, *p2, q1q2, d, dist)); + virial[0] += (*coulomb_u_kernel)(*p1, *p2, q1q2, d, dist); +#endif +#ifdef P3M + if (elc_kernel) + (*elc_kernel)(const_cast(*p1), const_cast(*p2), q1q2); +#endif // P3M + } +#endif // ELECTROSTATICS + + /***********************************************/ + /* thermostat */ + /***********************************************/ + + //std::cout << "Thermostat " << i << " " << j << "\n"; + /* The inter dpd force should not be part of the virial */ +#ifdef DPD + if (thermostat.thermo_switch & THERMO_DPD) { + auto const force = dpd_pair_force(*p1, *p2, *thermostat.dpd, box_geo, + ia_params, d, dist, dist2); + //p1.force() += force; + //p2.force() -= force; + pf += force; + } +#endif + + /***********************************************/ + /* short-range magnetostatics */ + /***********************************************/ + + //std::cout << "Magnetostatics " << i << " " << j << "\n"; +#ifdef DIPOLES + // real-space magnetic dipole-dipole + if (dipoles_kernel) { + pf += (*dipoles_kernel)(*p1, *p2, d, dist, dist2); + } +#endif + + Kokkos::atomic_add(&slice_force(i, 0), pf.f[0]); + Kokkos::atomic_add(&slice_force(i, 1), pf.f[1]); + Kokkos::atomic_add(&slice_force(i, 2), pf.f[2]); + Kokkos::atomic_add(&slice_torque(i, 0), pf.torque[0]); + Kokkos::atomic_add(&slice_torque(i, 1), pf.torque[1]); + Kokkos::atomic_add(&slice_torque(i, 2), pf.torque[2]); + + auto opf = calc_opposing_force(pf, d); + Kokkos::atomic_add(&slice_force(j, 0), opf.f[0]); + Kokkos::atomic_add(&slice_force(j, 1), opf.f[1]); + Kokkos::atomic_add(&slice_force(j, 2), opf.f[2]); + Kokkos::atomic_add(&slice_torque(j, 0), opf.torque[0]); + Kokkos::atomic_add(&slice_torque(j, 1), opf.torque[1]); + Kokkos::atomic_add(&slice_torque(j, 2), opf.torque[2]); + +#ifdef NPT + Kokkos::atomic_add(&virial_all(0), virial[0]); + Kokkos::atomic_add(&virial_all(1), virial[1]); + Kokkos::atomic_add(&virial_all(2), virial[2]); +#endif + +#ifdef COLLISION_DETECTION + //if (not collision_detection.is_off()) { + // collision_detection.detect_collision(*p1, *p2, dist2); + //} +#endif + }; + }; +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List"); +#endif + + // =================================================== + // Execute Kernel + // =================================================== +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Execute Kernel"); +#endif + Kokkos::RangePolicy policy(0, particle_storage.size()); + + FirstNeighborKernel first_neighbor_kernel(&cell_structure, bonded_ias, + nonbonded_ias, thermostat, box_geo, index_to_id, slice_position, slice_force, force_local_thread, slice_torque, slice_type, +#ifdef COLLISION_DETECTION + *collision_detection, +#endif + coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, + virial_all, num_threads, rank); + + //std::cout << rank << " " << index_to_id.size() << " Execute FirstNeighborKernel\n"; + // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" + // Feels like TeamOpTag is faster, atleast for large particle numbers + Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, + Cabana::FirstNeighborsTag(), + Cabana::TeamOpTag(), "verlet_list"); + + Kokkos::fence(); + +#ifdef NPT + Utils::Vector3d virial_vec{virial_all(0), virial_all(1), virial_all(2)}; + npt_add_virial_force_contribution(virial_vec); +#endif +#ifdef COLLISION_DETECTION + auto collision_kernel = [&](Particle const &p1, Particle const &p2, Distance const &d) { + if (not collision_detection->is_off()) { + collision_detection->detect_collision(p1, p2, d.dist2); + } + }; + cell_structure.non_bonded_loop(collision_kernel, verlet_criterion); +#endif + +#ifdef CALIPER + CALI_MARK_END("Cabana - Execute Kernel"); +#endif + + // =================================================== + // Add forces to particles + // =================================================== +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Particle Forces"); +#endif + for (auto & p : particles) { + auto const id = id_to_index.at(p.id()); + Utils::Vector3d f_vec{slice_force(id,0), slice_force(id, 1), slice_force(id, 2)}; + Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), slice_torque(id, 2)}; + + ParticleForce f(f_vec, torque_vec); + p.force_and_torque() += f; + } + + std::unordered_set processed_ids; + + for (auto & p : ghost_particles) { + int const pid = p.id(); + // Check if the particle has already been processed + if (processed_ids.find(pid) != processed_ids.end()) { + continue; + } + + // Check if the ghost particle is in the map, i.e. was used during force calculation + if (id_to_index.find(pid) == id_to_index.end()) { + continue; + } + + auto const id = id_to_index.at(pid); + + // Only add forces to ghost particles that are not as normal particles in the map, + // as they have already been added to the force calculation + if (id < particles.size()) { + continue; + } + + processed_ids.insert(pid); + + Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), slice_force(id, 2)}; + Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), slice_torque(id, 2)}; + + ParticleForce f(f_vec, torque_vec); + p.force_and_torque() += f; + } +#ifdef CALIPER + CALI_MARK_END("Cabana - Particle Forces"); +#endif + + } +} + +#endif diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index 56abf6c4d3a..5e863e764dc 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -94,6 +94,10 @@ System::System(Private) { min_global_cut = INACTIVE_CUTOFF; } +System::~System() { + cell_structure->reset_cabana_data(); +} + void System::initialize() { auto handle = shared_from_this(); cell_structure->bind_system(handle); diff --git a/src/core/system/System.hpp b/src/core/system/System.hpp index 97de937a119..585d6554034 100644 --- a/src/core/system/System.hpp +++ b/src/core/system/System.hpp @@ -86,6 +86,8 @@ class System : public std::enable_shared_from_this { static std::shared_ptr create(); + virtual ~System(); + #ifdef CUDA GpuParticleData gpu; #endif @@ -165,6 +167,7 @@ class System : public std::enable_shared_from_this { void npt_add_virial_contribution(double energy); void npt_add_virial_contribution(Utils::Vector3d const &force, Utils::Vector3d const &d); + void npt_add_virial_contribution(Utils::Vector3d const &virial); #endif // NPT /** @brief Calculate all forces. */ diff --git a/testsuite/python/caliper.py b/testsuite/python/caliper.py index 67aedbb11bc..16577b88899 100644 --- a/testsuite/python/caliper.py +++ b/testsuite/python/caliper.py @@ -49,6 +49,7 @@ @utx.skipIfMissingFeatures(["CALIPER"]) class Test(ut.TestCase): + @utx.skipIfExistingFeatures(["SHARED_MEMORY_PARALLELISM"]) @utx.skipIfMissingFeatures(["P3M", "WCA"]) def test_runtime_report(self): has_cuda = espressomd.has_features(["CUDA"]) diff --git a/testsuite/python/integrator_npt_stats.py b/testsuite/python/integrator_npt_stats.py index ed352fe1a1c..00a1c8eb51b 100644 --- a/testsuite/python/integrator_npt_stats.py +++ b/testsuite/python/integrator_npt_stats.py @@ -113,8 +113,8 @@ def test_compressibility_and_pressure(self): self.assertAlmostEqual(avp, p_ext, delta=0.02) self.assertAlmostEqual(compressibility, 0.5, delta=0.05) np.testing.assert_allclose(avp_sim_vir, avp_inst_vir, atol=1e-10) - self.assertAlmostEqual(avpV_sim, 100., delta=1.) - self.assertAlmostEqual(avpV_inst, 100., delta=1.) + self.assertAlmostEqual(avpV_sim, 100., delta=1.5) + self.assertAlmostEqual(avpV_inst, 100., delta=1.5) def test_negative_volume(self): """Test for NpT with bad parameters.""" diff --git a/testsuite/python/unittest_decorators.py b/testsuite/python/unittest_decorators.py index 32b988b80cf..19d1aa28ae1 100644 --- a/testsuite/python/unittest_decorators.py +++ b/testsuite/python/unittest_decorators.py @@ -79,3 +79,12 @@ def skipIfUnmetModuleVersionRequirement(module, version_requirement): return unittest.skip( "Skipping test: version requirement not met for module {}".format(module)) return no_skip + + +def skipIfExistingFeatures(*args): + """Unittest skipIf decorator for existing Espresso features.""" + if espressomd.has_features(*args): + return unittest.skip("Skipping test: existing feature") + return no_skip + + From 393c23f6d2b6ab5997207acdec8c4a85250bdf39 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 30 May 2025 19:40:56 +0200 Subject: [PATCH 02/94] Formatting --- src/core/cabana_data.hpp | 31 +- src/core/cell_system/CellStructure.cpp | 17 +- src/core/cell_system/CellStructure.hpp | 26 +- src/core/communication.cpp | 4 +- src/core/custom_verlet_list.hpp | 203 ++++++------- src/core/forces.cpp | 90 +++--- src/core/forces_inline.hpp | 1 - src/core/npt.cpp | 3 +- src/core/short_range_cabana.cpp | 380 +++++++++++++----------- src/core/system/System.cpp | 4 +- testsuite/python/unittest_decorators.py | 2 - 11 files changed, 381 insertions(+), 380 deletions(-) diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 8f538fbff39..a9b427d0887 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -21,8 +21,8 @@ #ifdef SHARED_MEMORY_PARALLELISM -#include #include "custom_verlet_list.hpp" +#include #include #include @@ -31,24 +31,27 @@ using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using ListAlgorithm = Cabana::HalfNeighborTag; -using ListType = Cabana::CustomVerletList; +using ListType = Cabana::CustomVerletList; class CabanaData { - ListType verlet_list; - std::unordered_map id_to_index; - std::vector index_to_id; + ListType verlet_list; + std::unordered_map id_to_index; + std::vector index_to_id; public: - CabanaData() = default; - CabanaData(ListType verlet_list, std::unordered_map id_to_index) + CabanaData() = default; + CabanaData(ListType verlet_list, std::unordered_map id_to_index) : verlet_list(verlet_list), id_to_index(id_to_index) {} - CabanaData(ListType verlet_list, std::unordered_map id_to_index, std::vector index_to_id) - : verlet_list(verlet_list), id_to_index(id_to_index), index_to_id(index_to_id) {} + CabanaData(ListType verlet_list, std::unordered_map id_to_index, + std::vector index_to_id) + : verlet_list(verlet_list), id_to_index(id_to_index), + index_to_id(index_to_id) {} - ListType get_verlet_list() const { return verlet_list; } - std::unordered_map get_id_to_index() const { return id_to_index; } - std::vector get_index_to_id() const { return index_to_id; } + ListType get_verlet_list() const { return verlet_list; } + std::unordered_map get_id_to_index() const { return id_to_index; } + std::vector get_index_to_id() const { return index_to_id; } - ~CabanaData() {}; + ~CabanaData() {}; }; -#endif +#endif diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index d0af7e7e8c8..54ed4ce44ed 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -52,9 +52,9 @@ #include #ifdef SHARED_MEMORY_PARALLELISM -#include -#include "custom_verlet_list.hpp" #include "cabana_data.hpp" +#include "custom_verlet_list.hpp" +#include #include #endif @@ -65,20 +65,16 @@ using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using ListAlgorithm = Cabana::HalfNeighborTag; -using ListType = Cabana::CustomVerletList; +using ListType = Cabana::CustomVerletList; - -CellStructure::~CellStructure() { - m_cabana_data.reset(); -} +CellStructure::~CellStructure() { m_cabana_data.reset(); } void CellStructure::set_cabana_data(std::unique_ptr data) { m_cabana_data = std::move(data); } -CabanaData& CellStructure::get_cabana_data() { - return *m_cabana_data; -} +CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } void CellStructure::reset_cabana_data() { m_rebuild_verlet_list = true; @@ -87,7 +83,6 @@ void CellStructure::reset_cabana_data() { #endif - CellStructure::CellStructure(BoxGeometry const &box) : m_decomposition{std::make_unique(box)} {} diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 249af45f8e9..be7f8adf34c 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -43,6 +43,7 @@ #include #include +#include #include #include #include @@ -54,7 +55,6 @@ #include #include #include -#include // forward declaration to not have to import cabana #ifdef SHARED_MEMORY_PARALLELISM @@ -290,7 +290,7 @@ struct CellStructure : public System::Leaf { ParticleRange ghost_particles() const { return Cells::particles(decomposition().ghost_cells()); } - + /** @brief whether to use parallel version of @ref for_each_local_particle */ bool use_parallel_for_each_local_particle() const { #ifdef SHARED_MEMORY_PARALLELISM @@ -661,27 +661,27 @@ struct CellStructure : public System::Leaf { #ifdef SHARED_MEMORY_PARALLELISM private: - std::unique_ptr m_cabana_data; + std::unique_ptr m_cabana_data; public: void set_cabana_data(std::unique_ptr data); - CabanaData& get_cabana_data(); + CabanaData &get_cabana_data(); void reset_cabana_data(); virtual ~CellStructure(); bool get_rebuild_verlet_list() const { return m_rebuild_verlet_list; } - bool get_rebuild_cabana_verlet_list() const { return m_rebuild_cabana_verlet_list; } + bool get_rebuild_cabana_verlet_list() const { + return m_rebuild_cabana_verlet_list; + } - template - void cabana_link_cell(Kernel kernel) { + template void cabana_link_cell(Kernel kernel) { auto const local_cells_span = decomposition().local_cells(); auto const first = boost::make_indirect_iterator(local_cells_span.begin()); auto const last = boost::make_indirect_iterator(local_cells_span.end()); - Algorithm::link_cell(first, last, [&kernel](Particle &p1, Particle &p2) { - kernel(p1, p2); - }); + Algorithm::link_cell( + first, last, [&kernel](Particle &p1, Particle &p2) { kernel(p1, p2); }); } template @@ -689,17 +689,17 @@ struct CellStructure : public System::Leaf { const VerletCriterion &verlet_criterion) { if (m_rebuild_cabana_verlet_list) { m_verlet_list.clear(); - + link_cell([&](Particle &p1, Particle &p2, Distance const &d) { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); } }); m_rebuild_cabana_verlet_list = false; - } + } for (auto const &pair : m_verlet_list) { kernel(*pair.first, *pair.second); - } + } } #endif diff --git a/src/core/communication.cpp b/src/core/communication.cpp index a6f56c37741..cc6bc693d4b 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -32,10 +32,10 @@ #endif #ifdef SHARED_MEMORY_PARALLELISM +#include "cell_system/CellStructure.hpp" +#include "system/System.hpp" #include #include -#include "system/System.hpp" -#include "cell_system/CellStructure.hpp" #endif #include diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 8a990104ab5..c7705d411f4 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -18,126 +18,119 @@ */ #pragma once -#ifdef SHARED_MEMORY_PARALLELISM +#ifdef SHARED_MEMORY_PARALLELISM #include -namespace Cabana -{ -// ONLY FOR 2D LAYOUT, OTHERWISE NEIGHBOR LIST INTERFACE IMPLEMENTATION WILL CAUSE PROBLEMS (NOT IMPLEMENTED) -template -class CustomVerletList : public VerletList -{ - public: - using Base = VerletList; - - // Default constructor - CustomVerletList() : Base() {} - - // Custom constructor - template - CustomVerletList(PositionSlice x, const std::size_t begin, const std::size_t end, - const std::size_t max_neigh) - { - initializeData(x.size(), max_neigh); - } - virtual ~CustomVerletList() {}; +namespace Cabana { +// ONLY FOR 2D LAYOUT, OTHERWISE NEIGHBOR LIST INTERFACE IMPLEMENTATION WILL +// CAUSE PROBLEMS (NOT IMPLEMENTED) +template +class CustomVerletList + : public VerletList { +public: + using Base = VerletList; + // Default constructor + CustomVerletList() : Base() {} -public: - Kokkos::View counts; - Kokkos::View neighbors; - - // Method to initialize _data without filling neighbors - KOKKOS_INLINE_FUNCTION - void initializeData(const std::size_t num_particles, const std::size_t max_neigh) - { - counts = Kokkos::View("num_neighbors", num_particles); - neighbors = Kokkos::View( - Kokkos::ViewAllocateWithoutInitializing("neighbors"), - num_particles, max_neigh); - } + // Custom constructor + template + CustomVerletList(PositionSlice x, const std::size_t begin, + const std::size_t end, const std::size_t max_neigh) { + initializeData(x.size(), max_neigh); + } + virtual ~CustomVerletList() {}; - // Method to dynamically expand the size of max_neighbors - KOKKOS_INLINE_FUNCTION - void expandMaxNeighbors(const std::size_t new_max_neigh) - { - // Create a new view with the larger size - Kokkos::View new_neighbors( - Kokkos::ViewAllocateWithoutInitializing("neighbors"), - neighbors.extent(0), new_max_neigh); - - // Copy existing data to the new view - Kokkos::parallel_for("copy_neighbors", neighbors.extent(0), KOKKOS_LAMBDA(const int i) { - for (std::size_t j = 0; j < counts(i); ++j) { - new_neighbors(i, j) = neighbors(i, j); - } +public: + Kokkos::View counts; + Kokkos::View neighbors; + + // Method to initialize _data without filling neighbors + KOKKOS_INLINE_FUNCTION + void initializeData(const std::size_t num_particles, + const std::size_t max_neigh) { + counts = Kokkos::View("num_neighbors", num_particles); + neighbors = Kokkos::View( + Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, + max_neigh); + } + + // Method to dynamically expand the size of max_neighbors + KOKKOS_INLINE_FUNCTION + void expandMaxNeighbors(const std::size_t new_max_neigh) { + // Create a new view with the larger size + Kokkos::View new_neighbors( + Kokkos::ViewAllocateWithoutInitializing("neighbors"), + neighbors.extent(0), new_max_neigh); + + // Copy existing data to the new view + Kokkos::parallel_for( + "copy_neighbors", neighbors.extent(0), KOKKOS_LAMBDA(const int i) { + for (std::size_t j = 0; j < counts(i); ++j) { + new_neighbors(i, j) = neighbors(i, j); + } }); - // Replace the old view with the new view - neighbors = new_neighbors; - } + // Replace the old view with the new view + neighbors = new_neighbors; + } - // Method to add a neighbor - KOKKOS_INLINE_FUNCTION - void addNeighbor(const int pid, const int nid) - { - std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); - if (count >= neighbors.extent(1)) { - expandMaxNeighbors(neighbors.extent(1) * 2); - } - neighbors(pid, count) = nid; + // Method to add a neighbor + KOKKOS_INLINE_FUNCTION + void addNeighbor(const int pid, const int nid) { + std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); + if (count >= neighbors.extent(1)) { + expandMaxNeighbors(neighbors.extent(1) * 2); } + neighbors(pid, count) = nid; + } }; template class NeighborList< - CustomVerletList> -{ - public: - //! Kokkos memory space. - using memory_space = MemorySpace; - //! Neighbor list type. - using list_type = - CustomVerletList; - - //! Get the total number of neighbors across all particles. - KOKKOS_INLINE_FUNCTION - static std::size_t totalNeighbor( const list_type& list ) - { - std::size_t num_p = list._data.counts.size(); - for ( std::size_t i = 0; i < num_p; ++i ) - num_p += list.counts( i ); - return num_p; - } - - //! Get the maximum number of neighbors per particle. - KOKKOS_INLINE_FUNCTION - static std::size_t maxNeighbor( const list_type& list ) - { - // Stored during neighbor search. - return list.max_n; - } - - //! Get the number of neighbors for a given particle index. - KOKKOS_INLINE_FUNCTION - static std::size_t numNeighbor( const list_type& list, - const std::size_t particle_index ) - { - return list.counts( particle_index ); - } - - //! Get the id for a neighbor for a given particle index and the index of - //! the neighbor relative to the particle. - KOKKOS_INLINE_FUNCTION - static std::size_t getNeighbor( const list_type& list, - const std::size_t particle_index, - const std::size_t count) - { - return list.neighbors( particle_index, count ); - } + CustomVerletList> { +public: + //! Kokkos memory space. + using memory_space = MemorySpace; + //! Neighbor list type. + using list_type = + CustomVerletList; + + //! Get the total number of neighbors across all particles. + KOKKOS_INLINE_FUNCTION + static std::size_t totalNeighbor(const list_type &list) { + std::size_t num_p = list._data.counts.size(); + for (std::size_t i = 0; i < num_p; ++i) + num_p += list.counts(i); + return num_p; + } + + //! Get the maximum number of neighbors per particle. + KOKKOS_INLINE_FUNCTION + static std::size_t maxNeighbor(const list_type &list) { + // Stored during neighbor search. + return list.max_n; + } + + //! Get the number of neighbors for a given particle index. + KOKKOS_INLINE_FUNCTION + static std::size_t numNeighbor(const list_type &list, + const std::size_t particle_index) { + return list.counts(particle_index); + } + + //! Get the id for a neighbor for a given particle index and the index of + //! the neighbor relative to the particle. + KOKKOS_INLINE_FUNCTION + static std::size_t getNeighbor(const list_type &list, + const std::size_t particle_index, + const std::size_t count) { + return list.neighbors(particle_index, count); + } }; -} +} // namespace Cabana #endif diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 9908e7320c9..feb05f99f6d 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -60,8 +60,8 @@ #endif #ifdef SHARED_MEMORY_PARALLELISM -#include #include "short_range_cabana.cpp" +#include #endif #include @@ -183,70 +183,62 @@ void System::System::calculate_forces() { #endif // interaction kernel is defined - auto bond_kernel = - [coulomb_kernel_ptr = get_ptr(coulomb_kernel), &bonded_ias = *bonded_ias, - &bond_breakage = *bond_breakage, &box_geo = *box_geo]( - Particle &p1, int bond_id, std::span partners) { - return add_bonded_force(p1, bond_id, partners, bonded_ias, - bond_breakage, box_geo, coulomb_kernel_ptr); - }; - auto pair_kernel = - [coulomb_kernel_ptr = get_ptr(coulomb_kernel), - dipoles_kernel_ptr = get_ptr(dipoles_kernel), - elc_kernel_ptr = get_ptr(elc_kernel), - coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel), - &nonbonded_ias = *nonbonded_ias, &thermostat = *thermostat, - &bonded_ias = *bonded_ias, + auto bond_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), + &bonded_ias = *bonded_ias, + &bond_breakage = *bond_breakage, + &box_geo = *box_geo](Particle &p1, int bond_id, + std::span partners) { + return add_bonded_force(p1, bond_id, partners, bonded_ias, bond_breakage, + box_geo, coulomb_kernel_ptr); + }; + auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), + dipoles_kernel_ptr = get_ptr(dipoles_kernel), + elc_kernel_ptr = get_ptr(elc_kernel), + coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel), + &nonbonded_ias = *nonbonded_ias, + &thermostat = *thermostat, &bonded_ias = *bonded_ias, #ifdef COLLISION_DETECTION - &collision_detection = *collision_detection, + &collision_detection = *collision_detection, #endif - &box_geo = *box_geo](Particle &p1, Particle &p2, Distance const &d) { - auto const &ia_params = - nonbonded_ias.get_ia_param(p1.type(), p2.type()); - add_non_bonded_pair_force(p1, p2, d.vec21, sqrt(d.dist2), d.dist2, - ia_params, thermostat, box_geo, bonded_ias, - coulomb_kernel_ptr, dipoles_kernel_ptr, - elc_kernel_ptr, coulomb_u_kernel_ptr); + &box_geo = *box_geo](Particle &p1, Particle &p2, + Distance const &d) { + auto const &ia_params = nonbonded_ias.get_ia_param(p1.type(), p2.type()); + add_non_bonded_pair_force(p1, p2, d.vec21, sqrt(d.dist2), d.dist2, + ia_params, thermostat, box_geo, bonded_ias, + coulomb_kernel_ptr, dipoles_kernel_ptr, + elc_kernel_ptr, coulomb_u_kernel_ptr); #ifdef COLLISION_DETECTION - if (not collision_detection.is_off()) { - collision_detection.detect_collision(p1, p2, d.dist2); - } + if (not collision_detection.is_off()) { + collision_detection.detect_collision(p1, p2, d.dist2); + } #endif - }; + }; #ifdef SHARED_MEMORY_PARALLELISM auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); - auto dipoles_kernel_ptr= get_ptr(dipoles_kernel); + auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); auto elc_kernel_ptr = get_ptr(elc_kernel); auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); cabana_short_range( - bond_kernel, - *bonded_ias, - coulomb_kernel_ptr, dipoles_kernel_ptr, elc_kernel_ptr, coulomb_u_kernel_ptr, + bond_kernel, *bonded_ias, coulomb_kernel_ptr, dipoles_kernel_ptr, + elc_kernel_ptr, coulomb_u_kernel_ptr, #ifdef COLLISION_DETECTION - collision_detection, + collision_detection, #endif - *cell_structure, - maximal_cutoff(), - bonded_ias->maximal_cutoff(), - *thermostat, - *box_geo, - *nonbonded_ias, - particles, - cell_structure->ghost_particles(), - VerletCriterion<>{*this, cell_structure->get_verlet_skin(), - get_interaction_range(), coulomb_cutoff, dipole_cutoff, - collision_detection_cutoff} - ); -#else - short_range_loop( - bond_kernel, - pair_kernel, *cell_structure, maximal_cutoff(), bonded_ias->maximal_cutoff(), + *thermostat, *box_geo, *nonbonded_ias, particles, + cell_structure->ghost_particles(), VerletCriterion<>{*this, cell_structure->get_verlet_skin(), get_interaction_range(), coulomb_cutoff, dipole_cutoff, collision_detection_cutoff}); - +#else + short_range_loop(bond_kernel, pair_kernel, *cell_structure, maximal_cutoff(), + bonded_ias->maximal_cutoff(), + VerletCriterion<>{*this, cell_structure->get_verlet_skin(), + get_interaction_range(), coulomb_cutoff, + dipole_cutoff, + collision_detection_cutoff}); + #endif constraints->add_forces(particles, get_sim_time()); oif_global->calculate_forces(); diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 00ac00c491b..1c73f4536a1 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -170,7 +170,6 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, return out; } - #ifdef SHARED_MEMORY_PARALLELISM using ReturnType = ParticleForce; #else diff --git a/src/core/npt.cpp b/src/core/npt.cpp index 623c4fb1a48..dcdaf080a13 100644 --- a/src/core/npt.cpp +++ b/src/core/npt.cpp @@ -149,7 +149,8 @@ void System::System::npt_add_virial_contribution(Utils::Vector3d const &force, } } -void System::System::npt_add_virial_contribution(Utils::Vector3d const &virial) { +void System::System::npt_add_virial_contribution( + Utils::Vector3d const &virial) { if ((propagation->integ_switch == INTEG_METHOD_NPT_ISO_AND) or (propagation->integ_switch == INTEG_METHOD_NPT_ISO_MTK)) { npt_inst_pressure->p_vir += virial; diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp index 5997264d75a..2ed689bee96 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.cpp @@ -29,18 +29,20 @@ #ifdef SHARED_MEMORY_PARALLELISM -#include #include "cabana_data.hpp" #include "custom_verlet_list.hpp" +#include #include +#include #include #include -#include - - template -inline void write_particle(Particle const &p, std::unordered_map const &id_to_index, SliceDouble3 &s_position, SliceDouble3 &s_force, SliceDouble3 &s_torque, SliceInt &s_id, SliceInt &s_type) { +inline void write_particle(Particle const &p, + std::unordered_map const &id_to_index, + SliceDouble3 &s_position, SliceDouble3 &s_force, + SliceDouble3 &s_torque, SliceInt &s_id, + SliceInt &s_type) { auto const pos = p.pos(); auto const id = id_to_index.at(p.id()); s_position(id, 0) = pos[0]; @@ -56,29 +58,29 @@ inline void write_particle(Particle const &p, std::unordered_map const s_torque(id, 2) = 0.0; } -template -void cabana_short_range(BondKernel bond_kernel, - [[maybe_unused]] BondedInteractionsMap const &bonded_ias, - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel, +template +void cabana_short_range( + BondKernel bond_kernel, + [[maybe_unused]] BondedInteractionsMap const &bonded_ias, + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel, #ifdef COLLISION_DETECTION - std::shared_ptr collision_detection, + std::shared_ptr collision_detection, #endif - CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, - Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, - InteractionsNonBonded &nonbonded_ias, - ParticleRange particles, ParticleRange ghost_particles, - VerletCriterion const &verlet_criterion = {}) { + CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, + Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, + InteractionsNonBonded &nonbonded_ias, ParticleRange particles, + ParticleRange ghost_particles, + VerletCriterion const &verlet_criterion = {}) { #ifdef CALIPER CALI_CXX_MARK_FUNCTION; #endif - #ifdef CALIPER +#ifdef CALIPER CALI_MARK_BEGIN("Espresso - Bond Kernel"); - #endif +#endif assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); @@ -86,9 +88,9 @@ void cabana_short_range(BondKernel bond_kernel, cell_structure.bond_loop(bond_kernel); } - #ifdef CALIPER - CALI_MARK_END("Espresso - Bond Kernel"); - #endif +#ifdef CALIPER + CALI_MARK_END("Espresso - Bond Kernel"); +#endif // Cabana short range loop if (pair_cutoff > 0.) { @@ -101,14 +103,16 @@ void cabana_short_range(BondKernel bond_kernel, CALI_MARK_BEGIN("Cabana - Setup"); #endif // Dont know where to do this better - using data_types = Cabana::MemberTypes; + using data_types = + Cabana::MemberTypes; using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using ListAlgorithm = Cabana::HalfNeighborTag; - using ListType = Cabana::CustomVerletList; + using ListType = Cabana::CustomVerletList; - //Number of threads + // Number of threads const int num_threads = execution_space().concurrency(); const int vector_length = 8; @@ -135,16 +139,17 @@ void cabana_short_range(BondKernel bond_kernel, saved_data = cell_structure.get_cabana_data(); } - // If we have to rebuild, we need to count the particles and create a new map + // If we have to rebuild, we need to count the particles and create a new + // map if (rebuild) { - - for (auto const& p : particles) { + + for (auto const &p : particles) { id_to_index[p.id()] = index; index_to_id.emplace_back(p.id()); index++; } - for (auto const& p : ghost_particles) { + for (auto const &p : ghost_particles) { if (not id_to_index.contains(p.id())) { id_to_index[p.id()] = index; index_to_id.emplace_back(p.id()); @@ -169,30 +174,35 @@ void cabana_short_range(BondKernel bond_kernel, #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Fill particle storage"); #endif - Cabana::AoSoA particle_storage("particles", number_of_unique_particles); + Cabana::AoSoA particle_storage( + "particles", number_of_unique_particles); auto slice_position = Cabana::slice<0>(particle_storage); auto slice_force = Cabana::slice<1>(particle_storage); auto slice_torque = Cabana::slice<2>(particle_storage); auto slice_id = Cabana::slice<3>(particle_storage); auto slice_type = Cabana::slice<4>(particle_storage); - for (auto const& p : particles) { - write_particle(p, id_to_index, slice_position, slice_force, slice_torque, slice_id, slice_type); + for (auto const &p : particles) { + write_particle(p, id_to_index, slice_position, slice_force, slice_torque, + slice_id, slice_type); } using TP = decltype(slice_position); using TF = decltype(slice_force); using TR = decltype(slice_torque); using TT = decltype(slice_type); - Kokkos::View virial_all("virial_all"); - Kokkos::View force_local_thread("force_local_thread", number_of_unique_particles, 3, num_threads); + Kokkos::View virial_all( + "virial_all"); + Kokkos::View force_local_thread( + "force_local_thread", number_of_unique_particles, 3, num_threads); - for (auto const& p : ghost_particles) { + for (auto const &p : ghost_particles) { // if the ghost is not in the previous map, but mpi moved it to this rank? // it will not have neighbors because we did not rebuild the verlet list. if (not id_to_index.contains(p.id())) { continue; } - write_particle(p, id_to_index, slice_position, slice_force, slice_torque, slice_id, slice_type); + write_particle(p, id_to_index, slice_position, slice_force, slice_torque, + slice_id, slice_type); } #ifdef CALIPER CALI_MARK_END("Cabana - Fill particle storage"); @@ -205,14 +215,15 @@ void cabana_short_range(BondKernel bond_kernel, CALI_MARK_BEGIN("Cabana - Verlet List"); #endif ListType verlet_list; - + // Rebuild verlet list if needed if (rebuild) { verlet_list = ListType(slice_position, 0, slice_position.size(), 64); - + auto kernel = [&](Particle const &p1, Particle const &p2) { - verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); + verlet_list.addNeighbor(id_to_index.at(p1.id()), + id_to_index.at(p2.id())); }; cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); @@ -229,7 +240,7 @@ void cabana_short_range(BondKernel bond_kernel, // fill customverletlist with pairs struct FirstNeighborKernel { - const CellStructure* cell; + const CellStructure *cell; [[maybe_unused]] const BondedInteractionsMap &bonded_ias; const InteractionsNonBonded &nonbonded_ias; const Thermostat::Thermostat &thermostat; @@ -237,11 +248,12 @@ void cabana_short_range(BondKernel bond_kernel, std::vector &index_to_id; TP &slice_position; TF &slice_force; - Kokkos::View force_local_thread; + Kokkos::View force_local_thread; TR &slice_torque; TT &slice_type; #ifdef COLLISION_DETECTION - //std::shared_ptr collision_detection; + // std::shared_ptr + // collision_detection; mutable CollisionDetection::CollisionDetection collision_detection; #endif Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; @@ -250,76 +262,76 @@ void cabana_short_range(BondKernel bond_kernel, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; Kokkos::View virial_all; - //Kokkos::View virial_all; + // Kokkos::View virial_all; int num_threads; int mpi_rank; - FirstNeighborKernel(const CellStructure* cell_, - [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, - const InteractionsNonBonded &nonbonded_ias_, - const Thermostat::Thermostat &thermostat_, - const BoxGeometry &box_geo_, - std::vector &index_to_id_, - TP &slice_position_, - TF &slice_force_, - Kokkos::View &force_local_thread_, - TR &slice_torque_, - TT &slice_type_, + FirstNeighborKernel( + const CellStructure *cell_, + [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, + const InteractionsNonBonded &nonbonded_ias_, + const Thermostat::Thermostat &thermostat_, + const BoxGeometry &box_geo_, std::vector &index_to_id_, + TP &slice_position_, TF &slice_force_, + Kokkos::View &force_local_thread_, TR &slice_torque_, + TT &slice_type_, #ifdef COLLISION_DETECTION - //std::shared_ptr collision_detection_, - CollisionDetection::CollisionDetection collision_detection_, -#endif - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, - Kokkos::View virial_all_, - //Kokkos::View virial_all_, - int num_threads_, - int mpi_rank_ - ) - : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - thermostat(thermostat_), box_geo(box_geo_), index_to_id(index_to_id_), - slice_position(slice_position_), slice_force(slice_force_), force_local_thread(force_local_thread_), - slice_torque(slice_torque_), slice_type(slice_type_), - collision_detection(collision_detection_), - coulomb_kernel(coulomb_kernel_), - dipoles_kernel(dipoles_kernel_), - elc_kernel(elc_kernel_), - coulomb_u_kernel(coulomb_u_kernel_), - virial_all(virial_all_), - num_threads(num_threads_), - mpi_rank(mpi_rank_) - {} + // std::shared_ptr + // collision_detection_, + CollisionDetection::CollisionDetection collision_detection_, +#endif + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const + *elc_kernel_, + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, + Kokkos::View virial_all_, + // Kokkos::View virial_all_, + int num_threads_, int mpi_rank_) + : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + thermostat(thermostat_), box_geo(box_geo_), + index_to_id(index_to_id_), slice_position(slice_position_), + slice_force(slice_force_), force_local_thread(force_local_thread_), + slice_torque(slice_torque_), slice_type(slice_type_), + collision_detection(collision_detection_), + coulomb_kernel(coulomb_kernel_), dipoles_kernel(dipoles_kernel_), + elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), + virial_all(virial_all_), num_threads(num_threads_), + mpi_rank(mpi_rank_) { + } KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { - Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), slice_position(i, 2)}; - Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), slice_position(j, 2)}; + Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), + slice_position(i, 2)}; + Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), + slice_position(j, 2)}; Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); auto const dist = d.norm(); auto const dist2 = dist * dist; - auto p1 = cell->get_local_particle(index_to_id.at(i)); - auto p2 = cell->get_local_particle(index_to_id.at(j)); - if (p1 == nullptr or p2 == nullptr) return; - //auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - //std::cout << thread_id << " " << index_to_id.size() << " Find " << p1 << " " << p2 << "\n"; - //std::cout << index_to_id.size() << " pos_i " << p1->pos() << "\n"; - //std::cout << index_to_id.size() << " pos_j " << p2->pos() << "\n"; - //if (dist > pair_cutoff) { - // return; - //} + auto p1 = cell->get_local_particle(index_to_id.at(i)); + auto p2 = cell->get_local_particle(index_to_id.at(j)); + if (p1 == nullptr or p2 == nullptr) + return; + // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); + // std::cout << thread_id << " " << index_to_id.size() << " Find " << p1 + // << " " << p2 << "\n"; std::cout << index_to_id.size() << " pos_i " << + // p1->pos() << "\n"; std::cout << index_to_id.size() << " pos_j " << + // p2->pos() << "\n"; if (dist > pair_cutoff) { + // return; + // } - IA_parameters const& ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); + IA_parameters const &ia_params = + nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); - ParticleForce pf{}; + ParticleForce pf{}; - /***********************************************/ - /* non-bonded pair potentials */ - /***********************************************/ + /***********************************************/ + /* non-bonded pair potentials */ + /***********************************************/ if (dist < ia_params.max_cut) { #ifdef EXCLUSIONS @@ -328,63 +340,65 @@ void cabana_short_range(BondKernel bond_kernel, pf += calc_central_radial_force(ia_params, d, dist); #ifdef THOLE pf.f += thole_pair_force(*p1, *p2, ia_params, d, dist, bonded_ias, - coulomb_kernel); + coulomb_kernel); #endif pf += calc_non_central_force(*p1, *p2, ia_params, d, dist); #ifdef EXCLUSIONS } #endif - } + } #ifdef NPT - //npt_add_virial_force_contribution(pf.f, d); - auto virial = hadamard_product(pf.f, d); - //auto virial = std::accumulate(virial_vec.begin(), virial_vec.end(), 0.0); + // npt_add_virial_force_contribution(pf.f, d); + auto virial = hadamard_product(pf.f, d); + // auto virial = std::accumulate(virial_vec.begin(), virial_vec.end(), + // 0.0); #endif #ifdef ELECTROSTATICS - // real-space electrostatic charge-charge interaction - auto const q1q2 = p1->q() * p2->q(); - if (q1q2 != 0. and coulomb_kernel != nullptr) { - pf.f += (*coulomb_kernel)(q1q2, d, dist); + // real-space electrostatic charge-charge interaction + auto const q1q2 = p1->q() * p2->q(); + if (q1q2 != 0. and coulomb_kernel != nullptr) { + pf.f += (*coulomb_kernel)(q1q2, d, dist); #ifdef NPT - //npt_add_virial_diagonalSum_contribution( - // (*coulomb_u_kernel)(*p1, *p2, q1q2, d, dist)); + // npt_add_virial_diagonalSum_contribution( + // (*coulomb_u_kernel)(*p1, *p2, q1q2, d, dist)); virial[0] += (*coulomb_u_kernel)(*p1, *p2, q1q2, d, dist); #endif #ifdef P3M - if (elc_kernel) - (*elc_kernel)(const_cast(*p1), const_cast(*p2), q1q2); + if (elc_kernel) + (*elc_kernel)(const_cast(*p1), + const_cast(*p2), q1q2); #endif // P3M } #endif // ELECTROSTATICS - /***********************************************/ - /* thermostat */ - /***********************************************/ + /***********************************************/ + /* thermostat */ + /***********************************************/ - //std::cout << "Thermostat " << i << " " << j << "\n"; - /* The inter dpd force should not be part of the virial */ + // std::cout << "Thermostat " << i << " " << j << "\n"; + /* The inter dpd force should not be part of the virial */ #ifdef DPD if (thermostat.thermo_switch & THERMO_DPD) { - auto const force = dpd_pair_force(*p1, *p2, *thermostat.dpd, box_geo, - ia_params, d, dist, dist2); - //p1.force() += force; - //p2.force() -= force; - pf += force; - } + auto const force = dpd_pair_force(*p1, *p2, *thermostat.dpd, box_geo, + ia_params, d, dist, dist2); + // p1.force() += force; + // p2.force() -= force; + pf += force; + } #endif - /***********************************************/ - /* short-range magnetostatics */ - /***********************************************/ + /***********************************************/ + /* short-range magnetostatics */ + /***********************************************/ - //std::cout << "Magnetostatics " << i << " " << j << "\n"; + // std::cout << "Magnetostatics " << i << " " << j << "\n"; #ifdef DIPOLES - // real-space magnetic dipole-dipole - if (dipoles_kernel) { - pf += (*dipoles_kernel)(*p1, *p2, d, dist, dist2); - } + // real-space magnetic dipole-dipole + if (dipoles_kernel) { + pf += (*dipoles_kernel)(*p1, *p2, d, dist, dist2); + } #endif Kokkos::atomic_add(&slice_force(i, 0), pf.f[0]); @@ -393,8 +407,8 @@ void cabana_short_range(BondKernel bond_kernel, Kokkos::atomic_add(&slice_torque(i, 0), pf.torque[0]); Kokkos::atomic_add(&slice_torque(i, 1), pf.torque[1]); Kokkos::atomic_add(&slice_torque(i, 2), pf.torque[2]); - - auto opf = calc_opposing_force(pf, d); + + auto opf = calc_opposing_force(pf, d); Kokkos::atomic_add(&slice_force(j, 0), opf.f[0]); Kokkos::atomic_add(&slice_force(j, 1), opf.f[1]); Kokkos::atomic_add(&slice_force(j, 2), opf.f[2]); @@ -403,15 +417,15 @@ void cabana_short_range(BondKernel bond_kernel, Kokkos::atomic_add(&slice_torque(j, 2), opf.torque[2]); #ifdef NPT - Kokkos::atomic_add(&virial_all(0), virial[0]); - Kokkos::atomic_add(&virial_all(1), virial[1]); - Kokkos::atomic_add(&virial_all(2), virial[2]); + Kokkos::atomic_add(&virial_all(0), virial[0]); + Kokkos::atomic_add(&virial_all(1), virial[1]); + Kokkos::atomic_add(&virial_all(2), virial[2]); #endif #ifdef COLLISION_DETECTION - //if (not collision_detection.is_off()) { - // collision_detection.detect_collision(*p1, *p2, dist2); - //} + // if (not collision_detection.is_off()) { + // collision_detection.detect_collision(*p1, *p2, dist2); + // } #endif }; }; @@ -427,20 +441,23 @@ void cabana_short_range(BondKernel bond_kernel, #endif Kokkos::RangePolicy policy(0, particle_storage.size()); - FirstNeighborKernel first_neighbor_kernel(&cell_structure, bonded_ias, - nonbonded_ias, thermostat, box_geo, index_to_id, slice_position, slice_force, force_local_thread, slice_torque, slice_type, + FirstNeighborKernel first_neighbor_kernel( + &cell_structure, bonded_ias, nonbonded_ias, thermostat, box_geo, + index_to_id, slice_position, slice_force, force_local_thread, + slice_torque, slice_type, #ifdef COLLISION_DETECTION - *collision_detection, + *collision_detection, #endif - coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, - virial_all, num_threads, rank); + coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, + virial_all, num_threads, rank); - //std::cout << rank << " " << index_to_id.size() << " Execute FirstNeighborKernel\n"; - // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" - // Feels like TeamOpTag is faster, atleast for large particle numbers + // std::cout << rank << " " << index_to_id.size() << " Execute + // FirstNeighborKernel\n"; + // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" + // Feels like TeamOpTag is faster, atleast for large particle numbers Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, - Cabana::FirstNeighborsTag(), - Cabana::TeamOpTag(), "verlet_list"); + Cabana::FirstNeighborsTag(), + Cabana::TeamOpTag(), "verlet_list"); Kokkos::fence(); @@ -449,7 +466,8 @@ void cabana_short_range(BondKernel bond_kernel, npt_add_virial_force_contribution(virial_vec); #endif #ifdef COLLISION_DETECTION - auto collision_kernel = [&](Particle const &p1, Particle const &p2, Distance const &d) { + auto collision_kernel = [&](Particle const &p1, Particle const &p2, + Distance const &d) { if (not collision_detection->is_off()) { collision_detection->detect_collision(p1, p2, d.dist2); } @@ -467,49 +485,53 @@ void cabana_short_range(BondKernel bond_kernel, #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Particle Forces"); #endif - for (auto & p : particles) { - auto const id = id_to_index.at(p.id()); - Utils::Vector3d f_vec{slice_force(id,0), slice_force(id, 1), slice_force(id, 2)}; - Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), slice_torque(id, 2)}; - - ParticleForce f(f_vec, torque_vec); - p.force_and_torque() += f; + for (auto &p : particles) { + auto const id = id_to_index.at(p.id()); + Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), + slice_force(id, 2)}; + Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), + slice_torque(id, 2)}; + + ParticleForce f(f_vec, torque_vec); + p.force_and_torque() += f; } std::unordered_set processed_ids; - for (auto & p : ghost_particles) { - int const pid = p.id(); - // Check if the particle has already been processed - if (processed_ids.find(pid) != processed_ids.end()) { - continue; - } + for (auto &p : ghost_particles) { + int const pid = p.id(); + // Check if the particle has already been processed + if (processed_ids.find(pid) != processed_ids.end()) { + continue; + } - // Check if the ghost particle is in the map, i.e. was used during force calculation - if (id_to_index.find(pid) == id_to_index.end()) { - continue; - } + // Check if the ghost particle is in the map, i.e. was used during force + // calculation + if (id_to_index.find(pid) == id_to_index.end()) { + continue; + } - auto const id = id_to_index.at(pid); + auto const id = id_to_index.at(pid); - // Only add forces to ghost particles that are not as normal particles in the map, - // as they have already been added to the force calculation - if (id < particles.size()) { - continue; - } + // Only add forces to ghost particles that are not as normal particles in + // the map, as they have already been added to the force calculation + if (id < particles.size()) { + continue; + } + + processed_ids.insert(pid); - processed_ids.insert(pid); + Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), + slice_force(id, 2)}; + Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), + slice_torque(id, 2)}; - Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), slice_force(id, 2)}; - Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), slice_torque(id, 2)}; - - ParticleForce f(f_vec, torque_vec); - p.force_and_torque() += f; + ParticleForce f(f_vec, torque_vec); + p.force_and_torque() += f; } #ifdef CALIPER CALI_MARK_END("Cabana - Particle Forces"); #endif - } } diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index 5e863e764dc..4aeeafa3046 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -94,9 +94,7 @@ System::System(Private) { min_global_cut = INACTIVE_CUTOFF; } -System::~System() { - cell_structure->reset_cabana_data(); -} +System::~System() { cell_structure->reset_cabana_data(); } void System::initialize() { auto handle = shared_from_this(); diff --git a/testsuite/python/unittest_decorators.py b/testsuite/python/unittest_decorators.py index 19d1aa28ae1..44f1fa9b664 100644 --- a/testsuite/python/unittest_decorators.py +++ b/testsuite/python/unittest_decorators.py @@ -86,5 +86,3 @@ def skipIfExistingFeatures(*args): if espressomd.has_features(*args): return unittest.skip("Skipping test: existing feature") return no_skip - - From 19f021eb51f7029a4d9c0afd5d6dc61b6ef4257b Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 30 May 2025 20:18:55 +0200 Subject: [PATCH 03/94] Corrected branching by macro variables --- src/core/system/System.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index 4aeeafa3046..79c59bb13c0 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -94,7 +94,11 @@ System::System(Private) { min_global_cut = INACTIVE_CUTOFF; } -System::~System() { cell_structure->reset_cabana_data(); } +System::~System() { +#ifdef SHARED_MEMORY_PARALLELISM + cell_structure->reset_cabana_data(); +#endif +} void System::initialize() { auto handle = shared_from_this(); From 85b9e2b10ab5993c6feb45dce27a4dfd1be66e3b Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 30 May 2025 20:33:25 +0200 Subject: [PATCH 04/94] Formatting --- src/core/system/System.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index 79c59bb13c0..cb5d7851da1 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -94,7 +94,7 @@ System::System(Private) { min_global_cut = INACTIVE_CUTOFF; } -System::~System() { +System::~System() { #ifdef SHARED_MEMORY_PARALLELISM cell_structure->reset_cabana_data(); #endif From 327727980427c4d8490083dcc0d4972bc0b33716 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 4 Jun 2025 17:45:40 +0200 Subject: [PATCH 05/94] For checking scalability --- CMakeLists.txt | 8 +- src/core/forces_inline.hpp | 18 ++- src/core/short_range_cabana.cpp | 278 +++++++++++++++++--------------- 3 files changed, 168 insertions(+), 136 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c21412c5401..2812fef30ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -545,8 +545,10 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) # cmake-format: on set(BUILD_SHARED_LIBS ON) set(CMAKE_SHARED_LIBRARY_PREFIX "lib") - set(Kokkos_ENABLE_SERIAL ON) - set(Kokkos_ENABLE_OPENMP ON) + #set(Kokkos_ENABLE_SERIAL ON) + #set(Kokkos_ENABLE_OPENMP ON) + set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "") + set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "") FetchContent_MakeAvailable(kokkos) set(BUILD_SHARED_LIBS ${ESPRESSO_BUILD_SHARED_LIBS_DEFAULT}) set(CMAKE_SHARED_LIBRARY_PREFIX "${ESPRESSO_SHARED_LIBRARY_PREFIX}") @@ -572,6 +574,7 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) cabana GIT_REPOSITORY https://github.com/ECP-copa/Cabana.git GIT_TAG ebfaa51 # 0.7.0 with patches + #GIT_TAG e76c1a15e1d68ce203d686bb68b0dfe41b2e1ad1 # latest PATCH_COMMAND patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cabana.patch ) # cmake-format: on @@ -922,6 +925,7 @@ if(ESPRESSO_BUILD_WITH_CALIPER) set(CALIPER_WITH_MPI on CACHE BOOL "") set(CALIPER_WITH_NVTX off CACHE BOOL "") set(CALIPER_WITH_CUPTI off CACHE BOOL "") + #set(CALIPER_WITH_OMPT on CACHE BOOL "") set(CALIPER_INSTALL_CONFIG off CACHE BOOL "") set(CALIPER_INSTALL_HEADERS off CACHE BOOL "") set(BUILD_SHARED_LIBS ON) diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 1c73f4536a1..3909e3b1827 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -171,7 +171,7 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, } #ifdef SHARED_MEMORY_PARALLELISM -using ReturnType = ParticleForce; +using ReturnType = std::pair; #else using ReturnType = void; #endif @@ -228,7 +228,11 @@ inline ReturnType add_non_bonded_pair_force( /* electrostatic is calculated by energy */ /*********************************************************************/ #ifdef NPT +#ifdef SHARED_MEMORY_PARALLELISM + auto virial = hadamard_product(pf.f, d); +#else npt_add_virial_force_contribution(pf.f, d); +#endif #endif /***********************************************/ @@ -241,9 +245,13 @@ inline ReturnType add_non_bonded_pair_force( if (q1q2 != 0. and coulomb_kernel != nullptr) { pf.f += (*coulomb_kernel)(q1q2, d, dist); #ifdef NPT +#ifdef SHARED_MEMORY_PARALLELISM + virial[0] += (*coulomb_u_kernel)(p1, p2, q1q2, d, dist); +#else npt_add_virial_diagonalSum_contribution( (*coulomb_u_kernel)(p1, p2, q1q2, d, dist)); -#endif +#endif //SHARED_MEMORY_PARALLELISM +#endif //NPT #ifdef P3M if (elc_kernel) (*elc_kernel)(p1, p2, q1q2); @@ -260,8 +268,12 @@ inline ReturnType add_non_bonded_pair_force( if (thermostat.thermo_switch & THERMO_DPD) { auto const force = dpd_pair_force(p1, p2, *thermostat.dpd, box_geo, ia_params, d, dist, dist2); +#ifdef SHARED_MEMORY_PARALLELISM + pf += force; +#else p1.force() += force; p2.force() -= force; +#endif } #endif @@ -281,7 +293,7 @@ inline ReturnType add_non_bonded_pair_force( /***********************************************/ #ifdef SHARED_MEMORY_PARALLELISM - return pf; + return std::pair{pf, virial}; #else p1.force_and_torque() += pf; p2.force_and_torque() += calc_opposing_force(pf, d); diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp index 2ed689bee96..65211ca350d 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.cpp @@ -32,10 +32,12 @@ #include "cabana_data.hpp" #include "custom_verlet_list.hpp" #include +#include #include #include #include #include +#include template inline void write_particle(Particle const &p, @@ -185,15 +187,21 @@ void cabana_short_range( write_particle(p, id_to_index, slice_position, slice_force, slice_torque, slice_id, slice_type); } - using TP = decltype(slice_position); - using TF = decltype(slice_force); - using TR = decltype(slice_torque); + //using TP = decltype(slice_position); + //using TF = decltype(slice_force); + //using TR = decltype(slice_torque); using TT = decltype(slice_type); Kokkos::View virial_all( "virial_all"); - Kokkos::View force_local_thread( - "force_local_thread", number_of_unique_particles, 3, num_threads); + + Kokkos::View local_force( + "local_force", num_threads, number_of_unique_particles, 3); + + Kokkos::View local_torque( + "local_torque", num_threads, number_of_unique_particles, 3); + + Kokkos::View local_virial("local_virial", num_threads, 3); for (auto const &p : ghost_particles) { // if the ghost is not in the previous map, but mpi moved it to this rank? @@ -219,7 +227,7 @@ void cabana_short_range( // Rebuild verlet list if needed if (rebuild) { - verlet_list = ListType(slice_position, 0, slice_position.size(), 64); + verlet_list = ListType(slice_position, 0, slice_position.size(), 32); auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), @@ -246,10 +254,9 @@ void cabana_short_range( const Thermostat::Thermostat &thermostat; const BoxGeometry &box_geo; std::vector &index_to_id; - TP &slice_position; - TF &slice_force; - Kokkos::View force_local_thread; - TR &slice_torque; + Kokkos::View local_force; + Kokkos::View local_torque; + Kokkos::View local_virial; TT &slice_type; #ifdef COLLISION_DETECTION // std::shared_ptr @@ -261,9 +268,6 @@ void cabana_short_range( Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; - Kokkos::View virial_all; - // Kokkos::View virial_all; - int num_threads; int mpi_rank; @@ -273,9 +277,10 @@ void cabana_short_range( const InteractionsNonBonded &nonbonded_ias_, const Thermostat::Thermostat &thermostat_, const BoxGeometry &box_geo_, std::vector &index_to_id_, - TP &slice_position_, TF &slice_force_, - Kokkos::View &force_local_thread_, TR &slice_torque_, - TT &slice_type_, + Kokkos::View local_force_, + Kokkos::View local_torque_, + Kokkos::View local_virial_, + TT &slice_type_, #ifdef COLLISION_DETECTION // std::shared_ptr // collision_detection_, @@ -286,140 +291,68 @@ void cabana_short_range( Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, - Kokkos::View virial_all_, - // Kokkos::View virial_all_, int num_threads_, int mpi_rank_) : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), thermostat(thermostat_), box_geo(box_geo_), - index_to_id(index_to_id_), slice_position(slice_position_), - slice_force(slice_force_), force_local_thread(force_local_thread_), - slice_torque(slice_torque_), slice_type(slice_type_), + index_to_id(index_to_id_), + local_force(local_force_), + local_torque(local_torque_), local_virial(local_virial_), + slice_type(slice_type_), collision_detection(collision_detection_), coulomb_kernel(coulomb_kernel_), dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), - virial_all(virial_all_), num_threads(num_threads_), - mpi_rank(mpi_rank_) { + num_threads(num_threads_), mpi_rank(mpi_rank_) { } KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { + /* Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), slice_position(i, 2)}; Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), slice_position(j, 2)}; - - Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); - auto const dist = d.norm(); - auto const dist2 = dist * dist; + */ auto p1 = cell->get_local_particle(index_to_id.at(i)); auto p2 = cell->get_local_particle(index_to_id.at(j)); if (p1 == nullptr or p2 == nullptr) return; + + Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), p2->pos()); + auto const dist = d.norm(); + auto const dist2 = dist * dist; + + auto thread_id = omp_get_thread_num(); // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - // std::cout << thread_id << " " << index_to_id.size() << " Find " << p1 - // << " " << p2 << "\n"; std::cout << index_to_id.size() << " pos_i " << - // p1->pos() << "\n"; std::cout << index_to_id.size() << " pos_j " << - // p2->pos() << "\n"; if (dist > pair_cutoff) { - // return; - // } + //std::cout << "in " << thread_id << " " << i << " " << j << "\n"; IA_parameters const &ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); - ParticleForce pf{}; - - /***********************************************/ - /* non-bonded pair potentials */ - /***********************************************/ - - if (dist < ia_params.max_cut) { -#ifdef EXCLUSIONS - if (do_nonbonded(*p1, *p2)) { -#endif - pf += calc_central_radial_force(ia_params, d, dist); -#ifdef THOLE - pf.f += thole_pair_force(*p1, *p2, ia_params, d, dist, bonded_ias, - coulomb_kernel); -#endif - pf += calc_non_central_force(*p1, *p2, ia_params, d, dist); -#ifdef EXCLUSIONS - } -#endif - } - -#ifdef NPT - // npt_add_virial_force_contribution(pf.f, d); - auto virial = hadamard_product(pf.f, d); - // auto virial = std::accumulate(virial_vec.begin(), virial_vec.end(), - // 0.0); -#endif - -#ifdef ELECTROSTATICS - // real-space electrostatic charge-charge interaction - auto const q1q2 = p1->q() * p2->q(); - if (q1q2 != 0. and coulomb_kernel != nullptr) { - pf.f += (*coulomb_kernel)(q1q2, d, dist); -#ifdef NPT - // npt_add_virial_diagonalSum_contribution( - // (*coulomb_u_kernel)(*p1, *p2, q1q2, d, dist)); - virial[0] += (*coulomb_u_kernel)(*p1, *p2, q1q2, d, dist); -#endif -#ifdef P3M - if (elc_kernel) - (*elc_kernel)(const_cast(*p1), - const_cast(*p2), q1q2); -#endif // P3M - } -#endif // ELECTROSTATICS - - /***********************************************/ - /* thermostat */ - /***********************************************/ - - // std::cout << "Thermostat " << i << " " << j << "\n"; - /* The inter dpd force should not be part of the virial */ -#ifdef DPD - if (thermostat.thermo_switch & THERMO_DPD) { - auto const force = dpd_pair_force(*p1, *p2, *thermostat.dpd, box_geo, - ia_params, d, dist, dist2); - // p1.force() += force; - // p2.force() -= force; - pf += force; - } -#endif + auto[pf, virial] = add_non_bonded_pair_force( + const_cast(*p1), const_cast(*p2), + d, dist, dist2, ia_params, thermostat, box_geo, bonded_ias, + coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); - /***********************************************/ - /* short-range magnetostatics */ - /***********************************************/ - - // std::cout << "Magnetostatics " << i << " " << j << "\n"; -#ifdef DIPOLES - // real-space magnetic dipole-dipole - if (dipoles_kernel) { - pf += (*dipoles_kernel)(*p1, *p2, d, dist, dist2); - } -#endif - - Kokkos::atomic_add(&slice_force(i, 0), pf.f[0]); - Kokkos::atomic_add(&slice_force(i, 1), pf.f[1]); - Kokkos::atomic_add(&slice_force(i, 2), pf.f[2]); - Kokkos::atomic_add(&slice_torque(i, 0), pf.torque[0]); - Kokkos::atomic_add(&slice_torque(i, 1), pf.torque[1]); - Kokkos::atomic_add(&slice_torque(i, 2), pf.torque[2]); + local_force(thread_id, i, 0) += pf.f[0]; + local_force(thread_id, i, 1) += pf.f[1]; + local_force(thread_id, i, 2) += pf.f[2]; + local_torque(thread_id, i, 0) += pf.torque[0]; + local_torque(thread_id, i, 1) += pf.torque[1]; + local_torque(thread_id, i, 2) += pf.torque[2]; auto opf = calc_opposing_force(pf, d); - Kokkos::atomic_add(&slice_force(j, 0), opf.f[0]); - Kokkos::atomic_add(&slice_force(j, 1), opf.f[1]); - Kokkos::atomic_add(&slice_force(j, 2), opf.f[2]); - Kokkos::atomic_add(&slice_torque(j, 0), opf.torque[0]); - Kokkos::atomic_add(&slice_torque(j, 1), opf.torque[1]); - Kokkos::atomic_add(&slice_torque(j, 2), opf.torque[2]); + local_force(thread_id, j, 0) += opf.f[0]; + local_force(thread_id, j, 1) += opf.f[1]; + local_force(thread_id, j, 2) += opf.f[2]; + local_torque(thread_id, j, 0) += opf.torque[0]; + local_torque(thread_id, j, 1) += opf.torque[1]; + local_torque(thread_id, j, 2) += opf.torque[2]; #ifdef NPT - Kokkos::atomic_add(&virial_all(0), virial[0]); - Kokkos::atomic_add(&virial_all(1), virial[1]); - Kokkos::atomic_add(&virial_all(2), virial[2]); + local_virial(thread_id, 0) += virial[0]; + local_virial(thread_id, 1) += virial[1]; + local_virial(thread_id, 2) += virial[2]; #endif #ifdef COLLISION_DETECTION @@ -443,28 +376,112 @@ void cabana_short_range( FirstNeighborKernel first_neighbor_kernel( &cell_structure, bonded_ias, nonbonded_ias, thermostat, box_geo, - index_to_id, slice_position, slice_force, force_local_thread, - slice_torque, slice_type, + index_to_id, + local_force, local_torque, local_virial, + slice_type, #ifdef COLLISION_DETECTION *collision_detection, #endif coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, - virial_all, num_threads, rank); - - // std::cout << rank << " " << index_to_id.size() << " Execute - // FirstNeighborKernel\n"; + num_threads, rank); + /* + // For using not custom_verletlist but Cabana::VeletList: + using s_ListType = Cabana::VerletList; + auto box_l = box_geo.length(); + double grid_min[3] = { 0.0, 0.0, 0.0 }; + double grid_max[3] = { box_l[0], box_l[1], box_l[2] }; + s_ListType s_verlet_list; + s_verlet_list = s_ListType(slice_position, 0, slice_position.size(), nonbonded_ias.maximal_cutoff(), 1.0, grid_min, grid_max); + */ + + //std::cout << rank << " " << num_threads << " Execute FirstNeighborKernel\n"; // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" // Feels like TeamOpTag is faster, atleast for large particle numbers Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), - Cabana::TeamOpTag(), "verlet_list"); + Cabana::SerialOpTag());//, "verlet_list"); + /* + // For checking how custom_verlet_list works: + using TeamPolicy = Kokkos::TeamPolicy>; + using MemberType = TeamPolicy::member_type; + + TeamPolicy team_policy(number_of_unique_particles, 1); + + using neighbor_list_traits = Cabana::NeighborList; + using s_neighbor_list_traits = Cabana::NeighborList; + + //std::cout << "Number of threads " << num_threads << "\n"; + Kokkos::parallel_for("force_calc_by_team", team_policy, + KOKKOS_LAMBDA(const MemberType& team_member) + { + const int i = team_member.league_rank(); // particle index in verlet_list + const int num_neighbors = neighbor_list_traits::numNeighbor(verlet_list, i); + //const int s_num_neighbors = s_neighbor_list_traits::numNeighbor(s_verlet_list, i); + //std::cout << "team_size " << team_member.team_size() << "\n"; + //std::cout << "neighbor " << i << " " << num_neighbors << " " << s_num_neighbors << "\n"; + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, num_neighbors), + [&](const int n) { + const int j = neighbor_list_traits::getNeighbor(verlet_list, i, n); // particle index in verlet_list + const int thread_id = omp_get_thread_num(); + std::cout << "tid " << thread_id << "\n"; + char region_name[64]; + sprintf(region_name, "work_region_thread_%d", thread_id); + cali_begin_region(region_name); + first_neighbor_kernel(i, j); + cali_end_region(region_name); + }); + }); + */ + Kokkos::fence(); + + Kokkos::parallel_for("reduce", policy, + KOKKOS_LAMBDA(const int i) { + double fx = 0.; + double fy = 0.; + double fz = 0.; + double tx = 0.; + double ty = 0.; + double tz = 0.; + for (int tid = 0; tid < num_threads; ++tid) { + fx += local_force(tid, i, 0); + fy += local_force(tid, i, 1); + fz += local_force(tid, i, 2); + tx += local_torque(tid, i, 0); + ty += local_torque(tid, i, 1); + tz += local_torque(tid, i, 2); + } + slice_force(i, 0) = fx; + slice_force(i, 1) = fy; + slice_force(i, 2) = fz; + slice_torque(i, 0) = tx; + slice_torque(i, 1) = ty; + slice_torque(i, 2) = tz; + } + ); Kokkos::fence(); #ifdef NPT - Utils::Vector3d virial_vec{virial_all(0), virial_all(1), virial_all(2)}; + double vx = 0.; + double vy = 0.; + double vz = 0.; + for (int tid = 0; tid < num_threads; ++tid) { + vx += local_virial(tid, 0); + vy += local_virial(tid, 1); + vz += local_virial(tid, 2); + } + Utils::Vector3d virial_vec{vx, vy, vz}; npt_add_virial_force_contribution(virial_vec); #endif +#ifdef CALIPER + CALI_MARK_END("Cabana - Execute Kernel"); +#endif + +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Collision Detection"); +#endif #ifdef COLLISION_DETECTION auto collision_kernel = [&](Particle const &p1, Particle const &p2, Distance const &d) { @@ -474,9 +491,8 @@ void cabana_short_range( }; cell_structure.non_bonded_loop(collision_kernel, verlet_criterion); #endif - #ifdef CALIPER - CALI_MARK_END("Cabana - Execute Kernel"); + CALI_MARK_END("Cabana - Collision Detection"); #endif // =================================================== From 6e87b84c405422c0f725c07add83a46b10adb339 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 00:02:36 +0200 Subject: [PATCH 06/94] Implemented VerletList by Cabana --- CMakeLists.txt | 10 +- src/core/cell_system/CellStructure.cpp | 4 + src/core/custom_verlet_list.hpp | 9 +- src/core/forces.cpp | 40 +-- src/core/forces_inline.hpp | 136 ++++++--- src/core/short_range_cabana.cpp | 383 ++++++++++++++++++------- 6 files changed, 418 insertions(+), 164 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2812fef30ed..78f2a1fdec4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -538,17 +538,15 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) FetchContent_Declare( kokkos GIT_REPOSITORY https://github.com/kokkos/kokkos.git - GIT_TAG aba6e3caf2b8814fe6764a5d27b7b181253df14f # version 4.5.1 - #GIT_TAG 18b830e # version 4.6.1 with patches + GIT_TAG 18b830e # version 4.6.1 with patches OVERRIDE_FIND_PACKAGE ) # cmake-format: on set(BUILD_SHARED_LIBS ON) set(CMAKE_SHARED_LIBRARY_PREFIX "lib") - #set(Kokkos_ENABLE_SERIAL ON) - #set(Kokkos_ENABLE_OPENMP ON) set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "") set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "") + set(Kokkos_ENABLE_IMPL_VIEW_LEGACY ON CACHE BOOL "") FetchContent_MakeAvailable(kokkos) set(BUILD_SHARED_LIBS ${ESPRESSO_BUILD_SHARED_LIBS_DEFAULT}) set(CMAKE_SHARED_LIBRARY_PREFIX "${ESPRESSO_SHARED_LIBRARY_PREFIX}") @@ -573,8 +571,8 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) FetchContent_Declare( cabana GIT_REPOSITORY https://github.com/ECP-copa/Cabana.git - GIT_TAG ebfaa51 # 0.7.0 with patches - #GIT_TAG e76c1a15e1d68ce203d686bb68b0dfe41b2e1ad1 # latest + #GIT_TAG ebfaa51 # 0.7.0 with patches + GIT_TAG e76c1a15e1d68ce203d686bb68b0dfe41b2e1ad1 # latest PATCH_COMMAND patch -p0 < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cabana.patch ) # cmake-format: on diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 54ed4ce44ed..95bba876b28 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -72,12 +72,15 @@ CellStructure::~CellStructure() { m_cabana_data.reset(); } void CellStructure::set_cabana_data(std::unique_ptr data) { m_cabana_data = std::move(data); + //m_rebuild_verlet_list = false; + //m_rebuild_cabana_verlet_list = false; } CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } void CellStructure::reset_cabana_data() { m_rebuild_verlet_list = true; + m_rebuild_cabana_verlet_list = true; m_cabana_data.reset(); } @@ -312,6 +315,7 @@ void CellStructure::set_verlet_skin(double value) { assert(value >= 0.); m_verlet_skin = value; m_verlet_skin_set = true; + m_rebuild_cabana_verlet_list = true; get_system().on_verlet_skin_change(); } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index c7705d411f4..488d69de7b1 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -67,7 +67,7 @@ class CustomVerletList // Copy existing data to the new view Kokkos::parallel_for( - "copy_neighbors", neighbors.extent(0), KOKKOS_LAMBDA(const int i) { + "copy_neighbors", neighbors.extent(0), [=, this](const int i) { for (std::size_t j = 0; j < counts(i); ++j) { new_neighbors(i, j) = neighbors(i, j); } @@ -101,10 +101,11 @@ class NeighborList< //! Get the total number of neighbors across all particles. KOKKOS_INLINE_FUNCTION static std::size_t totalNeighbor(const list_type &list) { - std::size_t num_p = list._data.counts.size(); + std::size_t total_n = 0; + std::size_t num_p = list.counts.size(); for (std::size_t i = 0; i < num_p; ++i) - num_p += list.counts(i); - return num_p; + total_n += list.counts(i); + return total_n; } //! Get the maximum number of neighbors per particle. diff --git a/src/core/forces.cpp b/src/core/forces.cpp index feb05f99f6d..07e42863cd0 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -191,6 +191,26 @@ void System::System::calculate_forces() { return add_bonded_force(p1, bond_id, partners, bonded_ias, bond_breakage, box_geo, coulomb_kernel_ptr); }; + +#ifdef SHARED_MEMORY_PARALLELISM + auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); + auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); + auto elc_kernel_ptr = get_ptr(elc_kernel); + auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); + cabana_short_range( + bond_kernel, *bonded_ias, coulomb_kernel_ptr, dipoles_kernel_ptr, + elc_kernel_ptr, coulomb_u_kernel_ptr, +#ifdef COLLISION_DETECTION + collision_detection, +#endif + *cell_structure, maximal_cutoff(), bonded_ias->maximal_cutoff(), + *thermostat, *box_geo, *nonbonded_ias, particles, + cell_structure->ghost_particles(), + VerletCriterion<>{*this, cell_structure->get_verlet_skin(), + get_interaction_range(), coulomb_cutoff, dipole_cutoff, + collision_detection_cutoff}); +#else + auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), dipoles_kernel_ptr = get_ptr(dipoles_kernel), elc_kernel_ptr = get_ptr(elc_kernel), @@ -203,7 +223,7 @@ void System::System::calculate_forces() { &box_geo = *box_geo](Particle &p1, Particle &p2, Distance const &d) { auto const &ia_params = nonbonded_ias.get_ia_param(p1.type(), p2.type()); - add_non_bonded_pair_force(p1, p2, d.vec21, sqrt(d.dist2), d.dist2, + add_non_bonded_pair_force(p1, p2, d.vec21, sqrt(d.dist2), d.dist2, p1.q()*p2.q(), ia_params, thermostat, box_geo, bonded_ias, coulomb_kernel_ptr, dipoles_kernel_ptr, elc_kernel_ptr, coulomb_u_kernel_ptr); @@ -214,24 +234,6 @@ void System::System::calculate_forces() { #endif }; -#ifdef SHARED_MEMORY_PARALLELISM - auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); - auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); - auto elc_kernel_ptr = get_ptr(elc_kernel); - auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); - cabana_short_range( - bond_kernel, *bonded_ias, coulomb_kernel_ptr, dipoles_kernel_ptr, - elc_kernel_ptr, coulomb_u_kernel_ptr, -#ifdef COLLISION_DETECTION - collision_detection, -#endif - *cell_structure, maximal_cutoff(), bonded_ias->maximal_cutoff(), - *thermostat, *box_geo, *nonbonded_ias, particles, - cell_structure->ghost_particles(), - VerletCriterion<>{*this, cell_structure->get_verlet_skin(), - get_interaction_range(), coulomb_cutoff, dipole_cutoff, - collision_detection_cutoff}); -#else short_range_loop(bond_kernel, pair_kernel, *cell_structure, maximal_cutoff(), bonded_ias->maximal_cutoff(), VerletCriterion<>{*this, cell_structure->get_verlet_skin(), diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 3909e3b1827..b72d2c579f2 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -75,6 +75,7 @@ #include #include #include +#include inline ParticleForce calc_central_radial_force(IA_parameters const &ia_params, Utils::Vector3d const &d, @@ -170,30 +171,50 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, return out; } -#ifdef SHARED_MEMORY_PARALLELISM -using ReturnType = std::pair; -#else -using ReturnType = void; +/** + * For the interaction which need NO particle information + */ +inline void add_non_bonded_pair_withot_p(ParticleForce &pf, Utils::Vector3d const &d, double dist, + double q1q2, IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel) { + + //ParticleForce pf{}; + + /***********************************************/ + /* non-bonded pair potentials */ + /***********************************************/ + + if (dist < ia_params.max_cut) { +#ifdef EXCLUSIONS + if (do_nonbonded) { #endif -/** Calculate non-bonded forces between a pair of particles and update their - * forces and torques. - * @param[in,out] p1 particle 1. - * @param[in,out] p2 particle 2. - * @param[in] d vector between @p p1 and @p p2. - * @param[in] dist distance between @p p1 and @p p2. - * @param[in] dist2 distance squared between @p p1 and @p p2. - * @param[in] ia_params non-bonded interaction kernels. - * @param[in] thermostat thermostat. - * @param[in] box_geo box geometry. - * @param[in] bonded_ias bonded interaction kernels. - * @param[in] coulomb_kernel Coulomb force kernel. - * @param[in] dipoles_kernel Dipolar force kernel. - * @param[in] elc_kernel ELC force correction kernel. - * @param[in] coulomb_u_kernel Coulomb energy kernel. + pf += calc_central_radial_force(ia_params, d, dist); +#ifdef EXCLUSIONS + } +#endif + } + + /***********************************************/ + /* short range cloumb potentials */ + /***********************************************/ + +#ifdef ELECTROSTATICS + // real-space electrostatic charge-charge interaction + //auto const q1q2 = p1.q() * p2.q(); + if (q1q2 != 0. and coulomb_kernel != nullptr) { + pf.f += (*coulomb_kernel)(q1q2, d, dist); + } +#endif // ELECTROSTATICS + //return pf; +} + + +/** + * For the interaction which need particle information */ -inline ReturnType add_non_bonded_pair_force( - Particle &p1, Particle &p2, Utils::Vector3d const &d, double dist, - double dist2, IA_parameters const &ia_params, +inline void add_non_bonded_pair_force_with_p( + Particle &p1, Particle &p2, ParticleForce &pf, Utils::Vector3d &virial, Utils::Vector3d const &d, double dist, + double dist2, double q1q2, IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, [[maybe_unused]] BondedInteractionsMap const &bonded_ias, Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, @@ -201,7 +222,8 @@ inline ReturnType add_non_bonded_pair_force( Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { - ParticleForce pf{}; + //ParticleForce pf{}; + //Utils::Vector3d virial{}; /***********************************************/ /* non-bonded pair potentials */ @@ -209,9 +231,9 @@ inline ReturnType add_non_bonded_pair_force( if (dist < ia_params.max_cut) { #ifdef EXCLUSIONS - if (do_nonbonded(p1, p2)) { + if (do_nonbonded) { #endif - pf += calc_central_radial_force(ia_params, d, dist); + //pf += calc_central_radial_force(ia_params, d, dist); #ifdef THOLE pf.f += thole_pair_force(p1, p2, ia_params, d, dist, bonded_ias, coulomb_kernel); @@ -229,9 +251,9 @@ inline ReturnType add_non_bonded_pair_force( /*********************************************************************/ #ifdef NPT #ifdef SHARED_MEMORY_PARALLELISM - auto virial = hadamard_product(pf.f, d); + virial += hadamard_product(pf.f, d); #else - npt_add_virial_force_contribution(pf.f, d); + npt_add_virial_force_contribution(pf.f + pf_n.f, d); #endif #endif @@ -241,9 +263,9 @@ inline ReturnType add_non_bonded_pair_force( #ifdef ELECTROSTATICS // real-space electrostatic charge-charge interaction - auto const q1q2 = p1.q() * p2.q(); + //auto const q1q2 = p1.q() * p2.q(); if (q1q2 != 0. and coulomb_kernel != nullptr) { - pf.f += (*coulomb_kernel)(q1q2, d, dist); + //pf.f += (*coulomb_kernel)(q1q2, d, dist); #ifdef NPT #ifdef SHARED_MEMORY_PARALLELISM virial[0] += (*coulomb_u_kernel)(p1, p2, q1q2, d, dist); @@ -268,12 +290,7 @@ inline ReturnType add_non_bonded_pair_force( if (thermostat.thermo_switch & THERMO_DPD) { auto const force = dpd_pair_force(p1, p2, *thermostat.dpd, box_geo, ia_params, d, dist, dist2); -#ifdef SHARED_MEMORY_PARALLELISM pf += force; -#else - p1.force() += force; - p2.force() -= force; -#endif } #endif @@ -287,6 +304,57 @@ inline ReturnType add_non_bonded_pair_force( pf += (*dipoles_kernel)(p1, p2, d, dist, dist2); } #endif + //return std::pair{pf, virial}; +} + +#ifdef SHARED_MEMORY_PARALLELISM +using ReturnType = std::pair; +#else +using ReturnType = void; +#endif +/** Calculate non-bonded forces between a pair of particles and update their + * forces and torques. + * @param[in,out] p1 particle 1. + * @param[in,out] p2 particle 2. + * @param[in] d vector between @p p1 and @p p2. + * @param[in] dist distance between @p p1 and @p p2. + * @param[in] dist2 distance squared between @p p1 and @p p2. + * @param[in] q1q2 charge x charge between @p p1 and @p p2. + * @param[in] ia_params non-bonded interaction kernels. + * @param[in] thermostat thermostat. + * @param[in] box_geo box geometry. + * @param[in] bonded_ias bonded interaction kernels. + * @param[in] coulomb_kernel Coulomb force kernel. + * @param[in] dipoles_kernel Dipolar force kernel. + * @param[in] elc_kernel ELC force correction kernel. + * @param[in] coulomb_u_kernel Coulomb energy kernel. + */ +inline ReturnType add_non_bonded_pair_force( + Particle &p1, Particle &p2, Utils::Vector3d const &d, double dist, + double dist2, double q1q2, IA_parameters const &ia_params, + Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, + [[maybe_unused]] BondedInteractionsMap const &bonded_ias, + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { + + ParticleForce pf{}; + Utils::Vector3d virial{}; + +#ifdef EXCLUSIONS + bool do_nonbonded_flag = do_nonbonded(p1, p2); +#else + bool do_nonbonded_flag = true; +#endif + + add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, coulomb_kernel); + +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or defined(DPD) or defined(DIPOLES) + add_non_bonded_pair_force_with_p( p1, p2, pf, virial, d, dist, + dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, + bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); +#endif /***********************************************/ /* add total non-bonded forces to particles */ diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp index 65211ca350d..424e3817695 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.cpp @@ -39,25 +39,56 @@ #include #include -template -inline void write_particle(Particle const &p, - std::unordered_map const &id_to_index, +inline double wrap1(double x, double L) { + auto result = x - std::floor(x / L) * L; + return result; +} + +inline double wrap2(double x, double L) { + auto result = x - std::floor(x / L) * L; + if (result >= L) result -= std::nextafter(L, 0.); + return result; +} + +inline bool contains(std::vector const &storage, int const value) { + return (std::find(storage.begin(), storage.end(), value) != storage.end()); +} + +template +inline void write_particle(Particle const &p, int const &id, SliceDouble3 &s_position, SliceDouble3 &s_force, - SliceDouble3 &s_torque, SliceInt &s_id, - SliceInt &s_type) { + SliceDouble3 &s_torque, SliceDouble &s_charge, + SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, Utils::Vector3d &box_l) { + //SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, BoxGeometry const &box_geo) { auto const pos = p.pos(); - auto const id = id_to_index.at(p.id()); - s_position(id, 0) = pos[0]; - s_position(id, 1) = pos[1]; - s_position(id, 2) = pos[2]; + s_position(id, 0) = wrap2(pos[0], box_l[0]); + s_position(id, 1) = wrap2(pos[1], box_l[1]); + s_position(id, 2) = wrap2(pos[2], box_l[2]); s_id(id) = p.id(); + s_charge(id) = p.q(); s_type(id) = p.type(); + s_ghost(id) = p.is_ghost(); s_force(id, 0) = 0.0; s_force(id, 1) = 0.0; s_force(id, 2) = 0.0; s_torque(id, 0) = 0.0; s_torque(id, 1) = 0.0; s_torque(id, 2) = 0.0; + assert(s_position(id, 0) >= 0. && s_position(id, 0) < box_l[0]); + assert(s_position(id, 1) >= 0. && s_position(id, 1) < box_l[1]); + assert(s_position(id, 2) >= 0. && s_position(id, 2) < box_l[2]); + /*if (p.id() == 325) { + std::cout << "0 CHECK 325 " + << pos[0] << " " + << pos[1] << " " + << pos[2] << "\n"; + } + if (p.id() == 512) { + std::cout << "0 CHECK 512 " + << pos[0] << " " + << pos[1] << " " + << pos[2] << "\n"; + }*/ } template @@ -83,7 +114,6 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Espresso - Bond Kernel"); #endif - assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); if (bond_cutoff >= 0.) { @@ -106,7 +136,7 @@ void cabana_short_range( #endif // Dont know where to do this better using data_types = - Cabana::MemberTypes; + Cabana::MemberTypes; using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -146,21 +176,22 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - id_to_index[p.id()] = index; + //id_to_index[p.id()] = index; index_to_id.emplace_back(p.id()); index++; } for (auto const &p : ghost_particles) { - if (not id_to_index.contains(p.id())) { - id_to_index[p.id()] = index; + //if (not id_to_index.contains(p.id())) { + // id_to_index[p.id()] = index; + if (not contains(index_to_id, p.id())) { index_to_id.emplace_back(p.id()); index++; } } } else { // If we do not rebuild we can use the saved map - id_to_index = saved_data.get_id_to_index(); + //id_to_index = saved_data.get_id_to_index(); index_to_id = saved_data.get_index_to_id(); index = id_to_index.size(); } @@ -181,20 +212,40 @@ void cabana_short_range( auto slice_position = Cabana::slice<0>(particle_storage); auto slice_force = Cabana::slice<1>(particle_storage); auto slice_torque = Cabana::slice<2>(particle_storage); - auto slice_id = Cabana::slice<3>(particle_storage); - auto slice_type = Cabana::slice<4>(particle_storage); + auto slice_charge = Cabana::slice<3>(particle_storage); + auto slice_id = Cabana::slice<4>(particle_storage); + auto slice_type = Cabana::slice<5>(particle_storage); + auto slice_ghost = Cabana::slice<6>(particle_storage); + auto box_l = box_geo.length(); + int p_id = 0; + std::vector registered_pid{}; + //std::vector ghost_pid{}; for (auto const &p : particles) { - write_particle(p, id_to_index, slice_position, slice_force, slice_torque, - slice_id, slice_type); + write_particle(p, p_id, slice_position, slice_force, slice_torque, + slice_charge, slice_id, slice_type, slice_ghost, box_l); + registered_pid.emplace_back(p.id()); + ++p_id; } - //using TP = decltype(slice_position); + for (auto const &p : ghost_particles) { + // if the ghost is not in the previous map, but mpi moved it to this rank? + // it will not have neighbors because we did not rebuild the verlet list. + if (contains(registered_pid, p.id())) { + continue; + } + write_particle(p, p_id, slice_position, slice_force, slice_torque, + slice_charge, slice_id, slice_type, slice_ghost, box_l); + registered_pid.emplace_back(p.id()); + //ghost_pid.emplace_back(p.id()); + ++p_id; + } + + using TP = decltype(slice_position); //using TF = decltype(slice_force); //using TR = decltype(slice_torque); + using TQ = decltype(slice_charge); + using TI = decltype(slice_id); using TT = decltype(slice_type); - Kokkos::View virial_all( - "virial_all"); - Kokkos::View local_force( "local_force", num_threads, number_of_unique_particles, 3); @@ -203,15 +254,6 @@ void cabana_short_range( Kokkos::View local_virial("local_virial", num_threads, 3); - for (auto const &p : ghost_particles) { - // if the ghost is not in the previous map, but mpi moved it to this rank? - // it will not have neighbors because we did not rebuild the verlet list. - if (not id_to_index.contains(p.id())) { - continue; - } - write_particle(p, id_to_index, slice_position, slice_force, slice_torque, - slice_id, slice_type); - } #ifdef CALIPER CALI_MARK_END("Cabana - Fill particle storage"); #endif @@ -223,40 +265,159 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - Verlet List"); #endif ListType verlet_list; + std::vector> pair_check; // Rebuild verlet list if needed if (rebuild) { - verlet_list = ListType(slice_position, 0, slice_position.size(), 32); - + verlet_list = ListType(slice_position, 0, slice_position.size(), 64); + /* auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); - }; + std::cout << "Cell_structure " + << id_to_index.at(p1.id()) << " " + << id_to_index.at(p2.id()) << " " + << p1.id() << " " + << p2.id() << " " + << p1.pos() << " " + << p2.pos() << "\n"; + if (p1.id() < p2.id()) { + pair_check.emplace_back(std::pair{p1.id(), p2.id()}); + } else { + pair_check.emplace_back(std::pair{p2.id(), p1.id()}); + } + };*/ - cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); + //cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); } else { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); } + // Creating LinkedCellList and VerletList: + Cabana::LinkedCellList cell_list; + double grid_min[3] = { 0.0, 0.0, 0.0 }; + double grid_max[3] = { box_l[0], box_l[1], box_l[2] }; + double grid_delta[3] = {}; + int cell_num[3] = {}; + double max_cutoff = System::get_system().get_interaction_range(); + for (int d = 0; d < 3; ++d) { + cell_num[d] = static_cast(box_l[d] / max_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + } + cell_list = Cabana::createLinkedCellList( + slice_position, grid_delta, grid_min, grid_max ); + //Now permute the AoSoA (i.e. reorder the data) using the linked cell list. + //Cabana::permute( cell_list, particle_storage ); + //ListType s_verlet_list; + if (rebuild && max_cutoff != INACTIVE_CUTOFF) { + //if (max_cutoff != INACTIVE_CUTOFF) { + verlet_list = ListType(slice_position, 0, slice_position.size(), 64); + for (int cid = 0; cid < cell_list.totalBins(); ++cid) { + cell_list(cid); + } + auto const particle_bins = cell_list.getParticleBins(); + //std::vector< std::vector< std::vector > > ijkIndexesInCell{}; + //for (int cid = 0; cid < cell_list.totalBins(); ++cid) { + std::vector< std::vector > ijkIndexes{}; + //int index[3] = {}; + //index[0] = static_cast(cid / (cell_num[1] * cell_num[2])); + //index[1] = static_cast((cid - index[0] * (cell_num[1] * cell_num[2])) / cell_num[2] ); + //index[2] = cid % cell_num[2]; + for (int n = 0; n < 27; ++n) { + std::vector dx = {0, 0, 0}; + dx[0] = static_cast(n / 9); + dx[1] = static_cast((n - 9*dx[0]) / 3); + dx[2] = n % 3; + //for (int d = 0; d < 3; ++d) { + // dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; + //} + ijkIndexes.emplace_back(dx); + } + //ijkIndexesInCell.emplace_back(ijkIndexes); + //} + // + auto const distance_function = + detail::MinimalImageDistance{std::as_const(cell_structure).decomposition().box()}; + + //auto kernel = [&cell_list, &particle_bins, &cell_num, &slice_id, &cell_structure, &verlet_criterion, &verlet_list, &distance_function](const int i) { + auto kernel = [&](const int i) { + + int index[3] = {}; + cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); + //auto ijkIndexes = ijkIndexesInCell[particle_bins(i)]; + int dx[3]; + for (int n = 0; n < 27; ++n) { + //auto dx = ijkIndexes[n]; + //auto relative_index = ijkIndexes[n]; + dx[0] = static_cast(n / 9); + dx[1] = static_cast((n - 9*dx[0]) / 3); + dx[2] = n % 3; + for (int d = 0; d < 3; ++d) { + dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; + } + + int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); + int size = cell_list.binSize(dx[0], dx[1], dx[2]); + + for (int j = offset; j < offset + size; j++) { + //int jj = j; + int jj = cell_list.permutation(j); + if (slice_id(i) < slice_id(jj)) { + auto p1 = cell_structure.get_local_particle(slice_id(i)); + auto p2 = cell_structure.get_local_particle(slice_id(jj)); + if (p1 == nullptr or p2 == nullptr) + continue; + if (p1->is_ghost()) { + //if (slice_ghost(slice_id(i))) { + //std::cout << slice_id(i) << " is ghost in rank " << rank << "\n"; + } else { + if ( verlet_criterion(*p1, *p2, distance_function(*p1, *p2)) ) { + verlet_list.addNeighbor(i, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << slice_id(i) << " " + << slice_id(j) << " " + << slice_position(i, 0) << " " + << slice_position(i, 1) << " " + << slice_position(i, 2) << " " + << slice_position(j, 0) << " " + << slice_position(j, 1) << " " + << slice_position(j, 2) << "\n";*/ + } + } + } + } + } + }; + + Kokkos::RangePolicy policy(0, particle_storage.size()); + Kokkos::parallel_for("calc_by_cell_list", policy, kernel); + Kokkos::fence(); + } + // Save data for next iteration if we just rebuilt if (rebuild) { - CabanaData new_data(verlet_list, id_to_index, index_to_id); + CabanaData new_data(verlet_list, id_to_index); cell_structure.set_cabana_data(std::make_unique(new_data)); } - // fill customverletlist with pairs + // calculate force with customverletlist struct FirstNeighborKernel { const CellStructure *cell; [[maybe_unused]] const BondedInteractionsMap &bonded_ias; const InteractionsNonBonded &nonbonded_ias; const Thermostat::Thermostat &thermostat; const BoxGeometry &box_geo; - std::vector &index_to_id; + //std::vector &index_to_id; Kokkos::View local_force; Kokkos::View local_torque; Kokkos::View local_virial; + TP &slice_position; + TQ &slice_charge; + TI &slice_id; TT &slice_type; #ifdef COLLISION_DETECTION // std::shared_ptr @@ -276,10 +437,14 @@ void cabana_short_range( [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, const Thermostat::Thermostat &thermostat_, - const BoxGeometry &box_geo_, std::vector &index_to_id_, + const BoxGeometry &box_geo_, + //std::vector &index_to_id_, Kokkos::View local_force_, Kokkos::View local_torque_, Kokkos::View local_virial_, + TP &slice_position_, + TQ &slice_charge_, + TI &slice_id_, TT &slice_type_, #ifdef COLLISION_DETECTION // std::shared_ptr @@ -294,11 +459,16 @@ void cabana_short_range( int num_threads_, int mpi_rank_) : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), thermostat(thermostat_), box_geo(box_geo_), - index_to_id(index_to_id_), + //index_to_id(index_to_id_), local_force(local_force_), local_torque(local_torque_), local_virial(local_virial_), + slice_position(slice_position_), + slice_charge(slice_charge_), + slice_id(slice_id_), slice_type(slice_type_), +#ifdef COLLISION_DETECTION collision_detection(collision_detection_), +#endif coulomb_kernel(coulomb_kernel_), dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), num_threads(num_threads_), mpi_rank(mpi_rank_) { @@ -306,34 +476,61 @@ void cabana_short_range( KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { - /* + Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), slice_position(i, 2)}; Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), slice_position(j, 2)}; - */ - auto p1 = cell->get_local_particle(index_to_id.at(i)); - auto p2 = cell->get_local_particle(index_to_id.at(j)); - if (p1 == nullptr or p2 == nullptr) - return; - - Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), p2->pos()); + Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); auto const dist = d.norm(); - auto const dist2 = dist * dist; + + auto const q1q2 = slice_charge(i) * slice_charge(j); auto thread_id = omp_get_thread_num(); // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - //std::cout << "in " << thread_id << " " << i << " " << j << "\n"; + //std::cout << "in " << thread_id << " " << i << " " << j << " " << q1q2 << "\n"; IA_parameters const &ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); + /* + auto p1 = cell->get_local_particle(slice_id(i)); + auto p2 = cell->get_local_particle(slice_id(j)); + + if (p1 == nullptr or p2 == nullptr) + return; auto[pf, virial] = add_non_bonded_pair_force( const_cast(*p1), const_cast(*p2), - d, dist, dist2, ia_params, thermostat, box_geo, bonded_ias, + d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); + */ + // + ParticleForce pf{}; + Utils::Vector3d virial{}; + +#ifdef EXCLUSIONS + bool do_nonbonded_flag = do_nonbonded(*p1, *p2); +#else + bool do_nonbonded_flag = true; +#endif + + add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, coulomb_kernel); + +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or defined(DPD) or defined(DIPOLES) + auto p1 = cell->get_local_particle(slice_id(i)); + auto p2 = cell->get_local_particle(slice_id(j)); + + if (p1 == nullptr or p2 == nullptr) + return; + auto const dist2 = dist * dist; + + add_non_bonded_pair_force_with_p( const_cast(*p1), const_cast(*p2), pf, virial, d, dist, + dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, + bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); +#endif + // local_force(thread_id, i, 0) += pf.f[0]; local_force(thread_id, i, 1) += pf.f[1]; local_force(thread_id, i, 2) += pf.f[2]; @@ -349,6 +546,18 @@ void cabana_short_range( local_torque(thread_id, j, 1) += opf.torque[1]; local_torque(thread_id, j, 2) += opf.torque[2]; + /*if (p1->id() == 512 || p2->id() == 512) { + std::cout << "2 CHECK FORCE i " + << i << " " << j << " " + << p1->id() << " " + << p2->id() << " " + << dist << " " + << p1->is_ghost() << " " + << p2->is_ghost() << " " + << pf.f[0] << " " + << pf.f[1] << " " + << pf.f[2] << "\n"; + }*/ #ifdef NPT local_virial(thread_id, 0) += virial[0]; local_virial(thread_id, 1) += virial[1]; @@ -376,24 +585,17 @@ void cabana_short_range( FirstNeighborKernel first_neighbor_kernel( &cell_structure, bonded_ias, nonbonded_ias, thermostat, box_geo, - index_to_id, + //index_to_id, local_force, local_torque, local_virial, + slice_position, + slice_charge, + slice_id, slice_type, #ifdef COLLISION_DETECTION *collision_detection, #endif coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, num_threads, rank); - /* - // For using not custom_verletlist but Cabana::VeletList: - using s_ListType = Cabana::VerletList; - auto box_l = box_geo.length(); - double grid_min[3] = { 0.0, 0.0, 0.0 }; - double grid_max[3] = { box_l[0], box_l[1], box_l[2] }; - s_ListType s_verlet_list; - s_verlet_list = s_ListType(slice_position, 0, slice_position.size(), nonbonded_ias.maximal_cutoff(), 1.0, grid_min, grid_max); - */ //std::cout << rank << " " << num_threads << " Execute FirstNeighborKernel\n"; // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" @@ -401,41 +603,9 @@ void cabana_short_range( Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), Cabana::SerialOpTag());//, "verlet_list"); - /* - // For checking how custom_verlet_list works: - using TeamPolicy = Kokkos::TeamPolicy>; - using MemberType = TeamPolicy::member_type; - - TeamPolicy team_policy(number_of_unique_particles, 1); - - using neighbor_list_traits = Cabana::NeighborList; - using s_neighbor_list_traits = Cabana::NeighborList; - - //std::cout << "Number of threads " << num_threads << "\n"; - Kokkos::parallel_for("force_calc_by_team", team_policy, - KOKKOS_LAMBDA(const MemberType& team_member) - { - const int i = team_member.league_rank(); // particle index in verlet_list - const int num_neighbors = neighbor_list_traits::numNeighbor(verlet_list, i); - //const int s_num_neighbors = s_neighbor_list_traits::numNeighbor(s_verlet_list, i); - //std::cout << "team_size " << team_member.team_size() << "\n"; - //std::cout << "neighbor " << i << " " << num_neighbors << " " << s_num_neighbors << "\n"; - - Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, num_neighbors), - [&](const int n) { - const int j = neighbor_list_traits::getNeighbor(verlet_list, i, n); // particle index in verlet_list - const int thread_id = omp_get_thread_num(); - std::cout << "tid " << thread_id << "\n"; - char region_name[64]; - sprintf(region_name, "work_region_thread_%d", thread_id); - cali_begin_region(region_name); - first_neighbor_kernel(i, j); - cali_end_region(region_name); - }); - }); - */ Kokkos::fence(); + //Force and Torque reduction Kokkos::parallel_for("reduce", policy, KOKKOS_LAMBDA(const int i) { double fx = 0.; @@ -458,9 +628,16 @@ void cabana_short_range( slice_torque(i, 0) = tx; slice_torque(i, 1) = ty; slice_torque(i, 2) = tz; + + /*if (slice_id(i) == 325 || slice_id(i) == 512) { + std::cout << "3 CHECK FORCE " + << slice_id(i) << " " + << slice_force(i, 0) << " " + << slice_force(i, 1) << " " + << slice_force(i, 2) << "\n"; + }*/ } ); - Kokkos::fence(); #ifdef NPT @@ -501,17 +678,20 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Particle Forces"); #endif - for (auto &p : particles) { - auto const id = id_to_index.at(p.id()); + for (auto id = 0; id < particle_storage.size(); ++id) { + auto p = cell_structure.get_local_particle(slice_id(id)); + if (p == nullptr) { + return; + } Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), slice_force(id, 2)}; Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), slice_torque(id, 2)}; ParticleForce f(f_vec, torque_vec); - p.force_and_torque() += f; + p->force_and_torque() += f; } - + /* std::unordered_set processed_ids; for (auto &p : ghost_particles) { @@ -545,6 +725,7 @@ void cabana_short_range( ParticleForce f(f_vec, torque_vec); p.force_and_torque() += f; } + */ #ifdef CALIPER CALI_MARK_END("Cabana - Particle Forces"); #endif From e2b4b5d1a87a4cf042e5562ba729f616e0b191c2 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 00:04:48 +0200 Subject: [PATCH 07/94] Formatting --- CMakeLists.txt | 2 +- src/core/cell_system/CellStructure.cpp | 4 +- src/core/custom_verlet_list.hpp | 12 +- src/core/forces.cpp | 8 +- src/core/forces_inline.hpp | 48 +-- src/core/short_range_cabana.cpp | 432 +++++++++++++------------ 6 files changed, 257 insertions(+), 249 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78f2a1fdec4..9bc73c4f9cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -923,7 +923,7 @@ if(ESPRESSO_BUILD_WITH_CALIPER) set(CALIPER_WITH_MPI on CACHE BOOL "") set(CALIPER_WITH_NVTX off CACHE BOOL "") set(CALIPER_WITH_CUPTI off CACHE BOOL "") - #set(CALIPER_WITH_OMPT on CACHE BOOL "") + # set(CALIPER_WITH_OMPT on CACHE BOOL "") set(CALIPER_INSTALL_CONFIG off CACHE BOOL "") set(CALIPER_INSTALL_HEADERS off CACHE BOOL "") set(BUILD_SHARED_LIBS ON) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 95bba876b28..d8b3a404fd2 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -72,8 +72,8 @@ CellStructure::~CellStructure() { m_cabana_data.reset(); } void CellStructure::set_cabana_data(std::unique_ptr data) { m_cabana_data = std::move(data); - //m_rebuild_verlet_list = false; - //m_rebuild_cabana_verlet_list = false; + // m_rebuild_verlet_list = false; + // m_rebuild_cabana_verlet_list = false; } CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 488d69de7b1..7be03cfad7a 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -66,12 +66,12 @@ class CustomVerletList neighbors.extent(0), new_max_neigh); // Copy existing data to the new view - Kokkos::parallel_for( - "copy_neighbors", neighbors.extent(0), [=, this](const int i) { - for (std::size_t j = 0; j < counts(i); ++j) { - new_neighbors(i, j) = neighbors(i, j); - } - }); + Kokkos::parallel_for("copy_neighbors", neighbors.extent(0), + [=, this](const int i) { + for (std::size_t j = 0; j < counts(i); ++j) { + new_neighbors(i, j) = neighbors(i, j); + } + }); // Replace the old view with the new view neighbors = new_neighbors; diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 07e42863cd0..27284c577d4 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -223,10 +223,10 @@ void System::System::calculate_forces() { &box_geo = *box_geo](Particle &p1, Particle &p2, Distance const &d) { auto const &ia_params = nonbonded_ias.get_ia_param(p1.type(), p2.type()); - add_non_bonded_pair_force(p1, p2, d.vec21, sqrt(d.dist2), d.dist2, p1.q()*p2.q(), - ia_params, thermostat, box_geo, bonded_ias, - coulomb_kernel_ptr, dipoles_kernel_ptr, - elc_kernel_ptr, coulomb_u_kernel_ptr); + add_non_bonded_pair_force( + p1, p2, d.vec21, sqrt(d.dist2), d.dist2, p1.q() * p2.q(), ia_params, + thermostat, box_geo, bonded_ias, coulomb_kernel_ptr, dipoles_kernel_ptr, + elc_kernel_ptr, coulomb_u_kernel_ptr); #ifdef COLLISION_DETECTION if (not collision_detection.is_off()) { collision_detection.detect_collision(p1, p2, d.dist2); diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index b72d2c579f2..ffb49a84eb5 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -72,10 +72,10 @@ #include +#include #include #include #include -#include inline ParticleForce calc_central_radial_force(IA_parameters const &ia_params, Utils::Vector3d const &d, @@ -174,11 +174,12 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, /** * For the interaction which need NO particle information */ -inline void add_non_bonded_pair_withot_p(ParticleForce &pf, Utils::Vector3d const &d, double dist, - double q1q2, IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, +inline void add_non_bonded_pair_withot_p( + ParticleForce &pf, Utils::Vector3d const &d, double dist, double q1q2, + IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel) { - //ParticleForce pf{}; + // ParticleForce pf{}; /***********************************************/ /* non-bonded pair potentials */ @@ -200,21 +201,21 @@ inline void add_non_bonded_pair_withot_p(ParticleForce &pf, Utils::Vector3d cons #ifdef ELECTROSTATICS // real-space electrostatic charge-charge interaction - //auto const q1q2 = p1.q() * p2.q(); + // auto const q1q2 = p1.q() * p2.q(); if (q1q2 != 0. and coulomb_kernel != nullptr) { pf.f += (*coulomb_kernel)(q1q2, d, dist); } #endif // ELECTROSTATICS - //return pf; + // return pf; } - /** * For the interaction which need particle information */ inline void add_non_bonded_pair_force_with_p( - Particle &p1, Particle &p2, ParticleForce &pf, Utils::Vector3d &virial, Utils::Vector3d const &d, double dist, - double dist2, double q1q2, IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, + Particle &p1, Particle &p2, ParticleForce &pf, Utils::Vector3d &virial, + Utils::Vector3d const &d, double dist, double dist2, double q1q2, + IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, [[maybe_unused]] BondedInteractionsMap const &bonded_ias, Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, @@ -222,8 +223,8 @@ inline void add_non_bonded_pair_force_with_p( Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { - //ParticleForce pf{}; - //Utils::Vector3d virial{}; + // ParticleForce pf{}; + // Utils::Vector3d virial{}; /***********************************************/ /* non-bonded pair potentials */ @@ -233,7 +234,7 @@ inline void add_non_bonded_pair_force_with_p( #ifdef EXCLUSIONS if (do_nonbonded) { #endif - //pf += calc_central_radial_force(ia_params, d, dist); + // pf += calc_central_radial_force(ia_params, d, dist); #ifdef THOLE pf.f += thole_pair_force(p1, p2, ia_params, d, dist, bonded_ias, coulomb_kernel); @@ -263,17 +264,17 @@ inline void add_non_bonded_pair_force_with_p( #ifdef ELECTROSTATICS // real-space electrostatic charge-charge interaction - //auto const q1q2 = p1.q() * p2.q(); + // auto const q1q2 = p1.q() * p2.q(); if (q1q2 != 0. and coulomb_kernel != nullptr) { - //pf.f += (*coulomb_kernel)(q1q2, d, dist); + // pf.f += (*coulomb_kernel)(q1q2, d, dist); #ifdef NPT #ifdef SHARED_MEMORY_PARALLELISM virial[0] += (*coulomb_u_kernel)(p1, p2, q1q2, d, dist); #else npt_add_virial_diagonalSum_contribution( (*coulomb_u_kernel)(p1, p2, q1q2, d, dist)); -#endif //SHARED_MEMORY_PARALLELISM -#endif //NPT +#endif // SHARED_MEMORY_PARALLELISM +#endif // NPT #ifdef P3M if (elc_kernel) (*elc_kernel)(p1, p2, q1q2); @@ -304,7 +305,7 @@ inline void add_non_bonded_pair_force_with_p( pf += (*dipoles_kernel)(p1, p2, d, dist, dist2); } #endif - //return std::pair{pf, virial}; + // return std::pair{pf, virial}; } #ifdef SHARED_MEMORY_PARALLELISM @@ -348,12 +349,15 @@ inline ReturnType add_non_bonded_pair_force( bool do_nonbonded_flag = true; #endif - add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, coulomb_kernel); + add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, + coulomb_kernel); -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or defined(DPD) or defined(DIPOLES) - add_non_bonded_pair_force_with_p( p1, p2, pf, virial, d, dist, - dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, - bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) + add_non_bonded_pair_force_with_p( + p1, p2, pf, virial, d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, + thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, + elc_kernel, coulomb_u_kernel); #endif /***********************************************/ diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp index 424e3817695..6118b81f891 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.cpp @@ -35,9 +35,9 @@ #include #include #include +#include #include #include -#include inline double wrap1(double x, double L) { auto result = x - std::floor(x / L) * L; @@ -46,7 +46,8 @@ inline double wrap1(double x, double L) { inline double wrap2(double x, double L) { auto result = x - std::floor(x / L) * L; - if (result >= L) result -= std::nextafter(L, 0.); + if (result >= L) + result -= std::nextafter(L, 0.); return result; } @@ -54,12 +55,15 @@ inline bool contains(std::vector const &storage, int const value) { return (std::find(storage.begin(), storage.end(), value) != storage.end()); } -template +template inline void write_particle(Particle const &p, int const &id, SliceDouble3 &s_position, SliceDouble3 &s_force, SliceDouble3 &s_torque, SliceDouble &s_charge, - SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, Utils::Vector3d &box_l) { - //SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, BoxGeometry const &box_geo) { + SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, + Utils::Vector3d &box_l) { + // SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, BoxGeometry const + // &box_geo) { auto const pos = p.pos(); s_position(id, 0) = wrap2(pos[0], box_l[0]); s_position(id, 1) = wrap2(pos[1], box_l[1]); @@ -79,15 +83,15 @@ inline void write_particle(Particle const &p, int const &id, assert(s_position(id, 2) >= 0. && s_position(id, 2) < box_l[2]); /*if (p.id() == 325) { std::cout << "0 CHECK 325 " - << pos[0] << " " - << pos[1] << " " - << pos[2] << "\n"; + << pos[0] << " " + << pos[1] << " " + << pos[2] << "\n"; } if (p.id() == 512) { std::cout << "0 CHECK 512 " - << pos[0] << " " - << pos[1] << " " - << pos[2] << "\n"; + << pos[0] << " " + << pos[1] << " " + << pos[2] << "\n"; }*/ } @@ -135,8 +139,8 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - Setup"); #endif // Dont know where to do this better - using data_types = - Cabana::MemberTypes; + using data_types = Cabana::MemberTypes; using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -176,14 +180,14 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - //id_to_index[p.id()] = index; + // id_to_index[p.id()] = index; index_to_id.emplace_back(p.id()); index++; } for (auto const &p : ghost_particles) { - //if (not id_to_index.contains(p.id())) { - // id_to_index[p.id()] = index; + // if (not id_to_index.contains(p.id())) { + // id_to_index[p.id()] = index; if (not contains(index_to_id, p.id())) { index_to_id.emplace_back(p.id()); index++; @@ -191,7 +195,7 @@ void cabana_short_range( } } else { // If we do not rebuild we can use the saved map - //id_to_index = saved_data.get_id_to_index(); + // id_to_index = saved_data.get_id_to_index(); index_to_id = saved_data.get_index_to_id(); index = id_to_index.size(); } @@ -219,11 +223,11 @@ void cabana_short_range( auto box_l = box_geo.length(); int p_id = 0; std::vector registered_pid{}; - //std::vector ghost_pid{}; + // std::vector ghost_pid{}; for (auto const &p : particles) { write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); - registered_pid.emplace_back(p.id()); + registered_pid.emplace_back(p.id()); ++p_id; } for (auto const &p : ghost_particles) { @@ -234,14 +238,14 @@ void cabana_short_range( } write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); - registered_pid.emplace_back(p.id()); - //ghost_pid.emplace_back(p.id()); + registered_pid.emplace_back(p.id()); + // ghost_pid.emplace_back(p.id()); ++p_id; } using TP = decltype(slice_position); - //using TF = decltype(slice_force); - //using TR = decltype(slice_torque); + // using TF = decltype(slice_force); + // using TR = decltype(slice_torque); using TQ = decltype(slice_charge); using TI = decltype(slice_id); using TT = decltype(slice_type); @@ -252,7 +256,8 @@ void cabana_short_range( Kokkos::View local_torque( "local_torque", num_threads, number_of_unique_particles, 3); - Kokkos::View local_virial("local_virial", num_threads, 3); + Kokkos::View local_virial("local_virial", + num_threads, 3); #ifdef CALIPER CALI_MARK_END("Cabana - Fill particle storage"); @@ -275,21 +280,21 @@ void cabana_short_range( auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); - std::cout << "Cell_structure " - << id_to_index.at(p1.id()) << " " - << id_to_index.at(p2.id()) << " " - << p1.id() << " " - << p2.id() << " " - << p1.pos() << " " - << p2.pos() << "\n"; - if (p1.id() < p2.id()) { - pair_check.emplace_back(std::pair{p1.id(), p2.id()}); - } else { - pair_check.emplace_back(std::pair{p2.id(), p1.id()}); - } + std::cout << "Cell_structure " + << id_to_index.at(p1.id()) << " " + << id_to_index.at(p2.id()) << " " + << p1.id() << " " + << p2.id() << " " + << p1.pos() << " " + << p2.pos() << "\n"; + if (p1.id() < p2.id()) { + pair_check.emplace_back(std::pair{p1.id(), p2.id()}); + } else { + pair_check.emplace_back(std::pair{p2.id(), p1.id()}); + } };*/ - //cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); + // cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); } else { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); @@ -297,8 +302,8 @@ void cabana_short_range( // Creating LinkedCellList and VerletList: Cabana::LinkedCellList cell_list; - double grid_min[3] = { 0.0, 0.0, 0.0 }; - double grid_max[3] = { box_l[0], box_l[1], box_l[2] }; + double grid_min[3] = {0.0, 0.0, 0.0}; + double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; double grid_delta[3] = {}; int cell_num[3] = {}; double max_cutoff = System::get_system().get_interaction_range(); @@ -307,90 +312,92 @@ void cabana_short_range( grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); } cell_list = Cabana::createLinkedCellList( - slice_position, grid_delta, grid_min, grid_max ); - //Now permute the AoSoA (i.e. reorder the data) using the linked cell list. - //Cabana::permute( cell_list, particle_storage ); - //ListType s_verlet_list; + slice_position, grid_delta, grid_min, grid_max); + // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. + // Cabana::permute( cell_list, particle_storage ); + // ListType s_verlet_list; if (rebuild && max_cutoff != INACTIVE_CUTOFF) { - //if (max_cutoff != INACTIVE_CUTOFF) { + // if (max_cutoff != INACTIVE_CUTOFF) { verlet_list = ListType(slice_position, 0, slice_position.size(), 64); for (int cid = 0; cid < cell_list.totalBins(); ++cid) { - cell_list(cid); + cell_list(cid); } auto const particle_bins = cell_list.getParticleBins(); - //std::vector< std::vector< std::vector > > ijkIndexesInCell{}; - //for (int cid = 0; cid < cell_list.totalBins(); ++cid) { - std::vector< std::vector > ijkIndexes{}; - //int index[3] = {}; - //index[0] = static_cast(cid / (cell_num[1] * cell_num[2])); - //index[1] = static_cast((cid - index[0] * (cell_num[1] * cell_num[2])) / cell_num[2] ); - //index[2] = cid % cell_num[2]; - for (int n = 0; n < 27; ++n) { - std::vector dx = {0, 0, 0}; - dx[0] = static_cast(n / 9); - dx[1] = static_cast((n - 9*dx[0]) / 3); - dx[2] = n % 3; - //for (int d = 0; d < 3; ++d) { - // dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; - //} - ijkIndexes.emplace_back(dx); - } - //ijkIndexesInCell.emplace_back(ijkIndexes); + // std::vector< std::vector< std::vector > > ijkIndexesInCell{}; + // for (int cid = 0; cid < cell_list.totalBins(); ++cid) { + std::vector> ijkIndexes{}; + // int index[3] = {}; + // index[0] = static_cast(cid / (cell_num[1] * cell_num[2])); + // index[1] = static_cast((cid - index[0] * (cell_num[1] * + // cell_num[2])) / cell_num[2] ); index[2] = cid % cell_num[2]; + for (int n = 0; n < 27; ++n) { + std::vector dx = {0, 0, 0}; + dx[0] = static_cast(n / 9); + dx[1] = static_cast((n - 9 * dx[0]) / 3); + dx[2] = n % 3; + // for (int d = 0; d < 3; ++d) { + // dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; + // } + ijkIndexes.emplace_back(dx); + } + // ijkIndexesInCell.emplace_back(ijkIndexes); //} // - auto const distance_function = - detail::MinimalImageDistance{std::as_const(cell_structure).decomposition().box()}; + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; - //auto kernel = [&cell_list, &particle_bins, &cell_num, &slice_id, &cell_structure, &verlet_criterion, &verlet_list, &distance_function](const int i) { + // auto kernel = [&cell_list, &particle_bins, &cell_num, &slice_id, + // &cell_structure, &verlet_criterion, &verlet_list, + // &distance_function](const int i) { auto kernel = [&](const int i) { - - int index[3] = {}; - cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); - //auto ijkIndexes = ijkIndexesInCell[particle_bins(i)]; - int dx[3]; - for (int n = 0; n < 27; ++n) { - //auto dx = ijkIndexes[n]; - //auto relative_index = ijkIndexes[n]; - dx[0] = static_cast(n / 9); - dx[1] = static_cast((n - 9*dx[0]) / 3); - dx[2] = n % 3; - for (int d = 0; d < 3; ++d) { - dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; - } - - int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); - int size = cell_list.binSize(dx[0], dx[1], dx[2]); - - for (int j = offset; j < offset + size; j++) { - //int jj = j; - int jj = cell_list.permutation(j); - if (slice_id(i) < slice_id(jj)) { - auto p1 = cell_structure.get_local_particle(slice_id(i)); - auto p2 = cell_structure.get_local_particle(slice_id(jj)); - if (p1 == nullptr or p2 == nullptr) - continue; - if (p1->is_ghost()) { - //if (slice_ghost(slice_id(i))) { - //std::cout << slice_id(i) << " is ghost in rank " << rank << "\n"; - } else { - if ( verlet_criterion(*p1, *p2, distance_function(*p1, *p2)) ) { - verlet_list.addNeighbor(i, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << slice_id(i) << " " - << slice_id(j) << " " - << slice_position(i, 0) << " " - << slice_position(i, 1) << " " - << slice_position(i, 2) << " " - << slice_position(j, 0) << " " - << slice_position(j, 1) << " " - << slice_position(j, 2) << "\n";*/ - } - } - } - } - } + int index[3] = {}; + cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); + // auto ijkIndexes = ijkIndexesInCell[particle_bins(i)]; + int dx[3]; + for (int n = 0; n < 27; ++n) { + // auto dx = ijkIndexes[n]; + // auto relative_index = ijkIndexes[n]; + dx[0] = static_cast(n / 9); + dx[1] = static_cast((n - 9 * dx[0]) / 3); + dx[2] = n % 3; + for (int d = 0; d < 3; ++d) { + dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; + } + + int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); + int size = cell_list.binSize(dx[0], dx[1], dx[2]); + + for (int j = offset; j < offset + size; j++) { + // int jj = j; + int jj = cell_list.permutation(j); + if (slice_id(i) < slice_id(jj)) { + auto p1 = cell_structure.get_local_particle(slice_id(i)); + auto p2 = cell_structure.get_local_particle(slice_id(jj)); + if (p1 == nullptr or p2 == nullptr) + continue; + if (p1->is_ghost()) { + // if (slice_ghost(slice_id(i))) { + // std::cout << slice_id(i) << " is ghost in rank " << rank << + // "\n"; + } else { + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(i, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << slice_id(i) << " " + << slice_id(j) << " " + << slice_position(i, 0) << " " + << slice_position(i, 1) << " " + << slice_position(i, 2) << " " + << slice_position(j, 0) << " " + << slice_position(j, 1) << " " + << slice_position(j, 2) << "\n";*/ + } + } + } + } + } }; Kokkos::RangePolicy policy(0, particle_storage.size()); @@ -411,7 +418,7 @@ void cabana_short_range( const InteractionsNonBonded &nonbonded_ias; const Thermostat::Thermostat &thermostat; const BoxGeometry &box_geo; - //std::vector &index_to_id; + // std::vector &index_to_id; Kokkos::View local_force; Kokkos::View local_torque; Kokkos::View local_virial; @@ -438,14 +445,11 @@ void cabana_short_range( const InteractionsNonBonded &nonbonded_ias_, const Thermostat::Thermostat &thermostat_, const BoxGeometry &box_geo_, - //std::vector &index_to_id_, + // std::vector &index_to_id_, Kokkos::View local_force_, - Kokkos::View local_torque_, - Kokkos::View local_virial_, - TP &slice_position_, - TQ &slice_charge_, - TI &slice_id_, - TT &slice_type_, + Kokkos::View local_torque_, + Kokkos::View local_virial_, TP &slice_position_, + TQ &slice_charge_, TI &slice_id_, TT &slice_type_, #ifdef COLLISION_DETECTION // std::shared_ptr // collision_detection_, @@ -459,19 +463,17 @@ void cabana_short_range( int num_threads_, int mpi_rank_) : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), thermostat(thermostat_), box_geo(box_geo_), - //index_to_id(index_to_id_), - local_force(local_force_), - local_torque(local_torque_), local_virial(local_virial_), - slice_position(slice_position_), - slice_charge(slice_charge_), - slice_id(slice_id_), - slice_type(slice_type_), + // index_to_id(index_to_id_), + local_force(local_force_), local_torque(local_torque_), + local_virial(local_virial_), slice_position(slice_position_), + slice_charge(slice_charge_), slice_id(slice_id_), + slice_type(slice_type_), #ifdef COLLISION_DETECTION collision_detection(collision_detection_), #endif coulomb_kernel(coulomb_kernel_), dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), - num_threads(num_threads_), mpi_rank(mpi_rank_) { + num_threads(num_threads_), mpi_rank(mpi_rank_) { } KOKKOS_INLINE_FUNCTION @@ -485,39 +487,42 @@ void cabana_short_range( Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); auto const dist = d.norm(); - auto const q1q2 = slice_charge(i) * slice_charge(j); + auto const q1q2 = slice_charge(i) * slice_charge(j); - auto thread_id = omp_get_thread_num(); + auto thread_id = omp_get_thread_num(); // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - //std::cout << "in " << thread_id << " " << i << " " << j << " " << q1q2 << "\n"; + // std::cout << "in " << thread_id << " " << i << " " << j << " " << + // q1q2 << "\n"; IA_parameters const &ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); - /* + /* auto p1 = cell->get_local_particle(slice_id(i)); auto p2 = cell->get_local_particle(slice_id(j)); if (p1 == nullptr or p2 == nullptr) return; - auto[pf, virial] = add_non_bonded_pair_force( - const_cast(*p1), const_cast(*p2), - d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, - coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); - */ - // - ParticleForce pf{}; - Utils::Vector3d virial{}; + auto[pf, virial] = add_non_bonded_pair_force( + const_cast(*p1), const_cast(*p2), + d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, + coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); + */ + // + ParticleForce pf{}; + Utils::Vector3d virial{}; #ifdef EXCLUSIONS - bool do_nonbonded_flag = do_nonbonded(*p1, *p2); + bool do_nonbonded_flag = do_nonbonded(*p1, *p2); #else - bool do_nonbonded_flag = true; + bool do_nonbonded_flag = true; #endif - add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, coulomb_kernel); + add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, + do_nonbonded_flag, coulomb_kernel); -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or defined(DPD) or defined(DIPOLES) +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) auto p1 = cell->get_local_particle(slice_id(i)); auto p2 = cell->get_local_particle(slice_id(j)); @@ -526,11 +531,13 @@ void cabana_short_range( auto const dist2 = dist * dist; - add_non_bonded_pair_force_with_p( const_cast(*p1), const_cast(*p2), pf, virial, d, dist, - dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, - bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); + add_non_bonded_pair_force_with_p( + const_cast(*p1), const_cast(*p2), pf, + virial, d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, + thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, + elc_kernel, coulomb_u_kernel); #endif - // + // local_force(thread_id, i, 0) += pf.f[0]; local_force(thread_id, i, 1) += pf.f[1]; local_force(thread_id, i, 2) += pf.f[2]; @@ -546,18 +553,18 @@ void cabana_short_range( local_torque(thread_id, j, 1) += opf.torque[1]; local_torque(thread_id, j, 2) += opf.torque[2]; - /*if (p1->id() == 512 || p2->id() == 512) { - std::cout << "2 CHECK FORCE i " - << i << " " << j << " " - << p1->id() << " " - << p2->id() << " " - << dist << " " - << p1->is_ghost() << " " - << p2->is_ghost() << " " - << pf.f[0] << " " - << pf.f[1] << " " - << pf.f[2] << "\n"; - }*/ + /*if (p1->id() == 512 || p2->id() == 512) { + std::cout << "2 CHECK FORCE i " + << i << " " << j << " " + << p1->id() << " " + << p2->id() << " " + << dist << " " + << p1->is_ghost() << " " + << p2->is_ghost() << " " + << pf.f[0] << " " + << pf.f[1] << " " + << pf.f[2] << "\n"; + }*/ #ifdef NPT local_virial(thread_id, 0) += virial[0]; local_virial(thread_id, 1) += virial[1]; @@ -585,70 +592,67 @@ void cabana_short_range( FirstNeighborKernel first_neighbor_kernel( &cell_structure, bonded_ias, nonbonded_ias, thermostat, box_geo, - //index_to_id, - local_force, local_torque, local_virial, - slice_position, - slice_charge, - slice_id, - slice_type, + // index_to_id, + local_force, local_torque, local_virial, slice_position, slice_charge, + slice_id, slice_type, #ifdef COLLISION_DETECTION *collision_detection, #endif coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, num_threads, rank); - //std::cout << rank << " " << num_threads << " Execute FirstNeighborKernel\n"; - // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" - // Feels like TeamOpTag is faster, atleast for large particle numbers + // std::cout << rank << " " << num_threads << " Execute + // FirstNeighborKernel\n"; + // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" + // Feels like TeamOpTag is faster, atleast for large particle numbers Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), - Cabana::SerialOpTag());//, "verlet_list"); + Cabana::SerialOpTag()); //, "verlet_list"); Kokkos::fence(); - //Force and Torque reduction - Kokkos::parallel_for("reduce", policy, - KOKKOS_LAMBDA(const int i) { - double fx = 0.; - double fy = 0.; - double fz = 0.; - double tx = 0.; - double ty = 0.; - double tz = 0.; - for (int tid = 0; tid < num_threads; ++tid) { - fx += local_force(tid, i, 0); - fy += local_force(tid, i, 1); - fz += local_force(tid, i, 2); - tx += local_torque(tid, i, 0); - ty += local_torque(tid, i, 1); - tz += local_torque(tid, i, 2); - } - slice_force(i, 0) = fx; - slice_force(i, 1) = fy; - slice_force(i, 2) = fz; - slice_torque(i, 0) = tx; - slice_torque(i, 1) = ty; - slice_torque(i, 2) = tz; - - /*if (slice_id(i) == 325 || slice_id(i) == 512) { - std::cout << "3 CHECK FORCE " - << slice_id(i) << " " - << slice_force(i, 0) << " " - << slice_force(i, 1) << " " - << slice_force(i, 2) << "\n"; - }*/ - } - ); + // Force and Torque reduction + Kokkos::parallel_for( + "reduce", policy, KOKKOS_LAMBDA(const int i) { + double fx = 0.; + double fy = 0.; + double fz = 0.; + double tx = 0.; + double ty = 0.; + double tz = 0.; + for (int tid = 0; tid < num_threads; ++tid) { + fx += local_force(tid, i, 0); + fy += local_force(tid, i, 1); + fz += local_force(tid, i, 2); + tx += local_torque(tid, i, 0); + ty += local_torque(tid, i, 1); + tz += local_torque(tid, i, 2); + } + slice_force(i, 0) = fx; + slice_force(i, 1) = fy; + slice_force(i, 2) = fz; + slice_torque(i, 0) = tx; + slice_torque(i, 1) = ty; + slice_torque(i, 2) = tz; + + /*if (slice_id(i) == 325 || slice_id(i) == 512) { + std::cout << "3 CHECK FORCE " + << slice_id(i) << " " + << slice_force(i, 0) << " " + << slice_force(i, 1) << " " + << slice_force(i, 2) << "\n"; + }*/ + }); Kokkos::fence(); #ifdef NPT - double vx = 0.; - double vy = 0.; - double vz = 0.; - for (int tid = 0; tid < num_threads; ++tid) { - vx += local_virial(tid, 0); - vy += local_virial(tid, 1); - vz += local_virial(tid, 2); - } + double vx = 0.; + double vy = 0.; + double vz = 0.; + for (int tid = 0; tid < num_threads; ++tid) { + vx += local_virial(tid, 0); + vy += local_virial(tid, 1); + vz += local_virial(tid, 2); + } Utils::Vector3d virial_vec{vx, vy, vz}; npt_add_virial_force_contribution(virial_vec); #endif @@ -681,7 +685,7 @@ void cabana_short_range( for (auto id = 0; id < particle_storage.size(); ++id) { auto p = cell_structure.get_local_particle(slice_id(id)); if (p == nullptr) { - return; + return; } Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), slice_force(id, 2)}; From 33a9ebb7e21e8f758a3ed22f1242f25f7f0b7fcc Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 00:17:08 +0200 Subject: [PATCH 08/94] Corrected macro variable --- src/core/short_range_cabana.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp index 6118b81f891..f05f447b306 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.cpp @@ -513,6 +513,12 @@ void cabana_short_range( Utils::Vector3d virial{}; #ifdef EXCLUSIONS + auto p1 = cell->get_local_particle(slice_id(i)); + auto p2 = cell->get_local_particle(slice_id(j)); + + if (p1 == nullptr or p2 == nullptr) + return; + bool do_nonbonded_flag = do_nonbonded(*p1, *p2); #else bool do_nonbonded_flag = true; @@ -523,14 +529,15 @@ void cabana_short_range( #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) + auto const dist2 = dist * dist; + +#ifndef EXCLUSIONS auto p1 = cell->get_local_particle(slice_id(i)); auto p2 = cell->get_local_particle(slice_id(j)); if (p1 == nullptr or p2 == nullptr) return; - - auto const dist2 = dist * dist; - +#endif add_non_bonded_pair_force_with_p( const_cast(*p1), const_cast(*p2), pf, virial, d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, From 21e6e9407c33d44bbe303ebf4889b0314cc59d3d Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 11:35:48 +0200 Subject: [PATCH 09/94] Corrected the case for without SHAREDMEMORY --- src/core/forces_inline.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index ffb49a84eb5..fdaac0b5925 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -254,7 +254,7 @@ inline void add_non_bonded_pair_force_with_p( #ifdef SHARED_MEMORY_PARALLELISM virial += hadamard_product(pf.f, d); #else - npt_add_virial_force_contribution(pf.f + pf_n.f, d); + npt_add_virial_force_contribution(pf.f, d); #endif #endif From 0232ef4908c1d3454fa2cf4bb46f03b2f12d2e89 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 13:27:41 +0200 Subject: [PATCH 10/94] Slightly modified the creation of VerletList --- src/core/forces_inline.hpp | 3 + src/core/short_range_cabana.cpp | 98 +++++++++++++++------------------ 2 files changed, 47 insertions(+), 54 deletions(-) diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index fdaac0b5925..1344db61cf6 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -341,7 +341,10 @@ inline ReturnType add_non_bonded_pair_force( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { ParticleForce pf{}; +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(SHARED_MEMORY_PARALLELISM) Utils::Vector3d virial{}; +#endif #ifdef EXCLUSIONS bool do_nonbonded_flag = do_nonbonded(p1, p2); diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp index f05f447b306..f8211c4c090 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.cpp @@ -227,6 +227,7 @@ void cabana_short_range( for (auto const &p : particles) { write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); + if (p.is_ghost()) std::cout << "WIRED!!!!!!!!!\n"; registered_pid.emplace_back(p.id()); ++p_id; } @@ -323,77 +324,66 @@ void cabana_short_range( cell_list(cid); } auto const particle_bins = cell_list.getParticleBins(); - // std::vector< std::vector< std::vector > > ijkIndexesInCell{}; - // for (int cid = 0; cid < cell_list.totalBins(); ++cid) { - std::vector> ijkIndexes{}; - // int index[3] = {}; - // index[0] = static_cast(cid / (cell_num[1] * cell_num[2])); - // index[1] = static_cast((cid - index[0] * (cell_num[1] * - // cell_num[2])) / cell_num[2] ); index[2] = cid % cell_num[2]; - for (int n = 0; n < 27; ++n) { - std::vector dx = {0, 0, 0}; - dx[0] = static_cast(n / 9); - dx[1] = static_cast((n - 9 * dx[0]) / 3); - dx[2] = n % 3; - // for (int d = 0; d < 3; ++d) { - // dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; - // } - ijkIndexes.emplace_back(dx); + + Kokkos::View bin_offset("bin_offset", cell_num[0], cell_num[1], cell_num[2]); + Kokkos::View bin_size("bin_size", cell_num[0], cell_num[1], cell_num[2]); + for (int cid = 0; cid < cell_list.totalBins(); ++cid) { + int dx[3] = {}; + dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); + dx[1] = + static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) + / cell_num[2] ); + dx[2] = cid % cell_num[2]; + bin_offset(dx[0], dx[1], dx[2]) = cell_list.binOffset(dx[0], dx[1], dx[2]); + bin_size(dx[0], dx[1], dx[2]) = cell_list.binSize(dx[0], dx[1], dx[2]); } - // ijkIndexesInCell.emplace_back(ijkIndexes); - //} - // + constexpr int ijkIndexes[27][3] = + {{-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, + { 0, -1, -1}, { 0, -1, 0}, { 0, -1, 1}, { 0, 0, -1}, { 0, 0, 0}, { 0, 0, 1}, { 0, 1, -1}, { 0, 1, 0}, { 0, 1, 1}, + { 1, -1, -1}, { 1, -1, 0}, { 1, -1, 1}, { 1, 0, -1}, { 1, 0, 0}, { 1, 0, 1}, { 1, 1, -1}, { 1, 1, 0}, { 1, 1, 1}}; + auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; - // auto kernel = [&cell_list, &particle_bins, &cell_num, &slice_id, - // &cell_structure, &verlet_criterion, &verlet_list, - // &distance_function](const int i) { auto kernel = [&](const int i) { + int id_i = slice_id(i); + if (slice_ghost(i)) return; + auto p1 = cell_structure.get_local_particle(id_i); + if (p1 == nullptr) return; int index[3] = {}; cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); - // auto ijkIndexes = ijkIndexesInCell[particle_bins(i)]; int dx[3]; for (int n = 0; n < 27; ++n) { - // auto dx = ijkIndexes[n]; - // auto relative_index = ijkIndexes[n]; - dx[0] = static_cast(n / 9); - dx[1] = static_cast((n - 9 * dx[0]) / 3); - dx[2] = n % 3; for (int d = 0; d < 3; ++d) { - dx[d] = (-1 + dx[d] + index[d] + cell_num[d]) % cell_num[d]; + dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; } - int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); - int size = cell_list.binSize(dx[0], dx[1], dx[2]); + //int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); + //int size = cell_list.binSize(dx[0], dx[1], dx[2]); + int offset = bin_offset(dx[0], dx[1], dx[2]); + int size = bin_size(dx[0], dx[1], dx[2]); for (int j = offset; j < offset + size; j++) { // int jj = j; int jj = cell_list.permutation(j); - if (slice_id(i) < slice_id(jj)) { - auto p1 = cell_structure.get_local_particle(slice_id(i)); - auto p2 = cell_structure.get_local_particle(slice_id(jj)); - if (p1 == nullptr or p2 == nullptr) + int id_j = slice_id(jj); + if (id_i < id_j) { + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) continue; - if (p1->is_ghost()) { - // if (slice_ghost(slice_id(i))) { - // std::cout << slice_id(i) << " is ghost in rank " << rank << - // "\n"; - } else { - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(i, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << slice_id(i) << " " - << slice_id(j) << " " - << slice_position(i, 0) << " " - << slice_position(i, 1) << " " - << slice_position(i, 2) << " " - << slice_position(j, 0) << " " - << slice_position(j, 1) << " " - << slice_position(j, 2) << "\n";*/ - } + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(i, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << slice_id(i) << " " + << slice_id(j) << " " + << slice_position(i, 0) << " " + << slice_position(i, 1) << " " + << slice_position(i, 2) << " " + << slice_position(j, 0) << " " + << slice_position(j, 1) << " " + << slice_position(j, 2) << "\n";*/ } } } From bd7bc1d6872be84f9f48ece477d5de2c7388c0dc Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 13:29:54 +0200 Subject: [PATCH 11/94] Formatting --- src/core/short_range_cabana.cpp | 66 ++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.cpp index f8211c4c090..e6f505a8dc1 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.cpp @@ -227,7 +227,8 @@ void cabana_short_range( for (auto const &p : particles) { write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); - if (p.is_ghost()) std::cout << "WIRED!!!!!!!!!\n"; + if (p.is_ghost()) + std::cout << "WIRED!!!!!!!!!\n"; registered_pid.emplace_back(p.id()); ++p_id; } @@ -325,31 +326,38 @@ void cabana_short_range( } auto const particle_bins = cell_list.getParticleBins(); - Kokkos::View bin_offset("bin_offset", cell_num[0], cell_num[1], cell_num[2]); - Kokkos::View bin_size("bin_size", cell_num[0], cell_num[1], cell_num[2]); + Kokkos::View bin_offset( + "bin_offset", cell_num[0], cell_num[1], cell_num[2]); + Kokkos::View bin_size( + "bin_size", cell_num[0], cell_num[1], cell_num[2]); for (int cid = 0; cid < cell_list.totalBins(); ++cid) { int dx[3] = {}; dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); - dx[1] = - static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) - / cell_num[2] ); + dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / + cell_num[2]); dx[2] = cid % cell_num[2]; - bin_offset(dx[0], dx[1], dx[2]) = cell_list.binOffset(dx[0], dx[1], dx[2]); + bin_offset(dx[0], dx[1], dx[2]) = + cell_list.binOffset(dx[0], dx[1], dx[2]); bin_size(dx[0], dx[1], dx[2]) = cell_list.binSize(dx[0], dx[1], dx[2]); } - constexpr int ijkIndexes[27][3] = - {{-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, - { 0, -1, -1}, { 0, -1, 0}, { 0, -1, 1}, { 0, 0, -1}, { 0, 0, 0}, { 0, 0, 1}, { 0, 1, -1}, { 0, 1, 0}, { 0, 1, 1}, - { 1, -1, -1}, { 1, -1, 0}, { 1, -1, 1}, { 1, 0, -1}, { 1, 0, 0}, { 1, 0, 1}, { 1, 1, -1}, { 1, 1, 0}, { 1, 1, 1}}; + constexpr int ijkIndexes[27][3] = { + {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, + {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, + {0, -1, 0}, {0, -1, 1}, {0, 0, -1}, {0, 0, 0}, {0, 0, 1}, + {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, + {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, + {1, 1, 0}, {1, 1, 1}}; auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; auto kernel = [&](const int i) { - int id_i = slice_id(i); - if (slice_ghost(i)) return; + int id_i = slice_id(i); + if (slice_ghost(i)) + return; auto p1 = cell_structure.get_local_particle(id_i); - if (p1 == nullptr) return; + if (p1 == nullptr) + return; int index[3] = {}; cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); int dx[3]; @@ -358,8 +366,8 @@ void cabana_short_range( dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; } - //int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); - //int size = cell_list.binSize(dx[0], dx[1], dx[2]); + // int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); + // int size = cell_list.binSize(dx[0], dx[1], dx[2]); int offset = bin_offset(dx[0], dx[1], dx[2]); int size = bin_size(dx[0], dx[1], dx[2]); @@ -371,19 +379,19 @@ void cabana_short_range( auto p2 = cell_structure.get_local_particle(id_j); if (p2 == nullptr) continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(i, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << slice_id(i) << " " - << slice_id(j) << " " - << slice_position(i, 0) << " " - << slice_position(i, 1) << " " - << slice_position(i, 2) << " " - << slice_position(j, 0) << " " - << slice_position(j, 1) << " " - << slice_position(j, 2) << "\n";*/ + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(i, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << slice_id(i) << " " + << slice_id(j) << " " + << slice_position(i, 0) << " " + << slice_position(i, 1) << " " + << slice_position(i, 2) << " " + << slice_position(j, 0) << " " + << slice_position(j, 1) << " " + << slice_position(j, 2) << "\n";*/ } } } From 76cbc5955f33dc778bc93b0694b9bb3649f8727b Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 23:43:28 +0200 Subject: [PATCH 12/94] Fixed bug --- src/core/custom_verlet_list.hpp | 7 +- src/core/forces.cpp | 2 +- ...ange_cabana.cpp => short_range_cabana.hpp} | 233 ++++++++++-------- 3 files changed, 132 insertions(+), 110 deletions(-) rename src/core/{short_range_cabana.cpp => short_range_cabana.hpp} (82%) diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 7be03cfad7a..df9919f9236 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -58,6 +58,10 @@ class CustomVerletList } // Method to dynamically expand the size of max_neighbors + // This function may be vaiolated Kokkos's rule. + // Kokkos::View should not be created in Kokkos::parallel. + // However, addNeighbor is used in the Kokkos::parallel and + // this function is called from addNeighbor. KOKKOS_INLINE_FUNCTION void expandMaxNeighbors(const std::size_t new_max_neigh) { // Create a new view with the larger size @@ -82,7 +86,8 @@ class CustomVerletList void addNeighbor(const int pid, const int nid) { std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); if (count >= neighbors.extent(1)) { - expandMaxNeighbors(neighbors.extent(1) * 2); + //expandMaxNeighbors(neighbors.extent(1) * 2); + throw std::runtime_error("Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; } diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 27284c577d4..e40e37ef25d 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -60,7 +60,7 @@ #endif #ifdef SHARED_MEMORY_PARALLELISM -#include "short_range_cabana.cpp" +#include "short_range_cabana.hpp" #include #endif diff --git a/src/core/short_range_cabana.cpp b/src/core/short_range_cabana.hpp similarity index 82% rename from src/core/short_range_cabana.cpp rename to src/core/short_range_cabana.hpp index e6f505a8dc1..1cab9386fc4 100644 --- a/src/core/short_range_cabana.cpp +++ b/src/core/short_range_cabana.hpp @@ -22,6 +22,7 @@ #include "config/config.hpp" #include "cell_system/CellStructure.hpp" +#include "lees_edwards/lees_edwards.hpp" #ifdef CALIPER #include @@ -39,12 +40,7 @@ #include #include -inline double wrap1(double x, double L) { - auto result = x - std::floor(x / L) * L; - return result; -} - -inline double wrap2(double x, double L) { +inline double wrap(double x, double L) { auto result = x - std::floor(x / L) * L; if (result >= L) result -= std::nextafter(L, 0.); @@ -62,12 +58,10 @@ inline void write_particle(Particle const &p, int const &id, SliceDouble3 &s_torque, SliceDouble &s_charge, SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, Utils::Vector3d &box_l) { - // SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, BoxGeometry const - // &box_geo) { auto const pos = p.pos(); - s_position(id, 0) = wrap2(pos[0], box_l[0]); - s_position(id, 1) = wrap2(pos[1], box_l[1]); - s_position(id, 2) = wrap2(pos[2], box_l[2]); + s_position(id, 0) = wrap(pos[0], box_l[0]); + s_position(id, 1) = wrap(pos[1], box_l[1]); + s_position(id, 2) = wrap(pos[2], box_l[2]); s_id(id) = p.id(); s_charge(id) = p.q(); s_type(id) = p.type(); @@ -81,18 +75,6 @@ inline void write_particle(Particle const &p, int const &id, assert(s_position(id, 0) >= 0. && s_position(id, 0) < box_l[0]); assert(s_position(id, 1) >= 0. && s_position(id, 1) < box_l[1]); assert(s_position(id, 2) >= 0. && s_position(id, 2) < box_l[2]); - /*if (p.id() == 325) { - std::cout << "0 CHECK 325 " - << pos[0] << " " - << pos[1] << " " - << pos[2] << "\n"; - } - if (p.id() == 512) { - std::cout << "0 CHECK 512 " - << pos[0] << " " - << pos[1] << " " - << pos[2] << "\n"; - }*/ } template @@ -162,7 +144,7 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Index map"); #endif - std::unordered_map id_to_index{}; + std::unordered_map id_to_index{}; // For DEBUG std::vector index_to_id{}; int index = 0; @@ -180,22 +162,22 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - // id_to_index[p.id()] = index; - index_to_id.emplace_back(p.id()); + id_to_index[p.id()] = index; + //index_to_id.emplace_back(p.id()); index++; } for (auto const &p : ghost_particles) { - // if (not id_to_index.contains(p.id())) { - // id_to_index[p.id()] = index; - if (not contains(index_to_id, p.id())) { - index_to_id.emplace_back(p.id()); + if (not id_to_index.contains(p.id())) { + id_to_index[p.id()] = index; + //if (not contains(index_to_id, p.id())) { + //index_to_id.emplace_back(p.id()); index++; } } } else { // If we do not rebuild we can use the saved map - // id_to_index = saved_data.get_id_to_index(); + id_to_index = saved_data.get_id_to_index(); index_to_id = saved_data.get_index_to_id(); index = id_to_index.size(); } @@ -223,7 +205,6 @@ void cabana_short_range( auto box_l = box_geo.length(); int p_id = 0; std::vector registered_pid{}; - // std::vector ghost_pid{}; for (auto const &p : particles) { write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); @@ -241,7 +222,6 @@ void cabana_short_range( write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); registered_pid.emplace_back(p.id()); - // ghost_pid.emplace_back(p.id()); ++p_id; } @@ -272,55 +252,75 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - Verlet List"); #endif ListType verlet_list; - std::vector> pair_check; // Rebuild verlet list if needed + auto const &system = ::System::get_system(); + double max_cutoff = system.get_interaction_range(); + int max_counts = static_cast(27*max_cutoff*max_cutoff*max_cutoff/3); + if (max_counts < 64) max_counts = 64; if (rebuild) { - - verlet_list = ListType(slice_position, 0, slice_position.size(), 64); - /* - auto kernel = [&](Particle const &p1, Particle const &p2) { + verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); + /*auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); - std::cout << "Cell_structure " + //std::cout << "Cell_structure " << id_to_index.at(p1.id()) << " " << id_to_index.at(p2.id()) << " " << p1.id() << " " << p2.id() << " " << p1.pos() << " " - << p2.pos() << "\n"; - if (p1.id() < p2.id()) { - pair_check.emplace_back(std::pair{p1.id(), p2.id()}); - } else { - pair_check.emplace_back(std::pair{p2.id(), p1.id()}); - } - };*/ + << p2.pos() << "\n";// + //if (p1.id() < p2.id()) { + // pair_check.emplace_back(std::pair{p1.id(), p2.id()}); + //} else { + // pair_check.emplace_back(std::pair{p2.id(), p1.id()}); + //} + }; - // cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); + cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);*/ } else { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); } // Creating LinkedCellList and VerletList: + // Box Properties Cabana::LinkedCellList cell_list; double grid_min[3] = {0.0, 0.0, 0.0}; double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; double grid_delta[3] = {}; int cell_num[3] = {}; - double max_cutoff = System::get_system().get_interaction_range(); + double eff_cutoff; for (int d = 0; d < 3; ++d) { - cell_num[d] = static_cast(box_l[d] / max_cutoff); + eff_cutoff = max_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); } + // Lees-Edwards boundary condition + double le_offset; + int le_direction; + int le_normal; + int delta_lebc[3] = {0, 0, 0}; + auto le_protocol = system.lees_edwards->get_protocol(); + if (le_protocol == nullptr) { + le_offset = 0.; + le_direction = -1; + le_normal = -1; + } else { + le_offset = box_geo.lees_edwards_bc().pos_offset; + le_direction = box_geo.lees_edwards_bc().shear_direction; + le_normal = box_geo.lees_edwards_bc().shear_plane_normal; + delta_lebc[le_direction] = + static_cast(std::ceil(le_offset/grid_delta[le_direction])) % cell_num[le_direction]; + } cell_list = Cabana::createLinkedCellList( slice_position, grid_delta, grid_min, grid_max); // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. // Cabana::permute( cell_list, particle_storage ); - // ListType s_verlet_list; if (rebuild && max_cutoff != INACTIVE_CUTOFF) { - // if (max_cutoff != INACTIVE_CUTOFF) { - verlet_list = ListType(slice_position, 0, slice_position.size(), 64); + verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); for (int cid = 0; cid < cell_list.totalBins(); ++cid) { cell_list(cid); } @@ -352,6 +352,8 @@ void cabana_short_range( std::as_const(cell_structure).decomposition().box()}; auto kernel = [&](const int i) { + //auto kernel = [&slice_id, &slice_ghost, &cell_structure, &particle_bins, &cell_list, &ijkIndexes, &cell_num, + // &le_protocol, &le_direction, &le_normal, &delta_lebc, &bin_offset, &bin_size, &verlet_criterion, &distance_function, &verlet_list](const int i) { int id_i = slice_id(i); if (slice_ghost(i)) return; @@ -362,39 +364,78 @@ void cabana_short_range( cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); int dx[3]; for (int n = 0; n < 27; ++n) { + bool duplicate_cell = false; for (int d = 0; d < 3; ++d) { dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; + if (cell_num[d] <= 2 && ijkIndexes[n][d] + index[d] != dx[d]) + duplicate_cell = true; } - - // int offset = cell_list.binOffset(dx[0], dx[1], dx[2]); - // int size = cell_list.binSize(dx[0], dx[1], dx[2]); - int offset = bin_offset(dx[0], dx[1], dx[2]); - int size = bin_size(dx[0], dx[1], dx[2]); - - for (int j = offset; j < offset + size; j++) { - // int jj = j; - int jj = cell_list.permutation(j); - int id_j = slice_id(jj); - if (id_i < id_j) { - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) - continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(i, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << slice_id(i) << " " - << slice_id(j) << " " - << slice_position(i, 0) << " " - << slice_position(i, 1) << " " - << slice_position(i, 2) << " " - << slice_position(j, 0) << " " - << slice_position(j, 1) << " " - << slice_position(j, 2) << "\n";*/ - } - } - } + if (duplicate_cell) continue; + + //Lees-Edwards BC + int le_crossing = 0; + if (le_protocol != nullptr) { + le_crossing = + ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + cell_num[le_direction]) % cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + cell_num[le_direction]) % cell_num[le_direction]; + } + } + + int cell_offset = bin_offset(dx[0], dx[1], dx[2]); + int cell_size = bin_size(dx[0], dx[1], dx[2]); + + auto verlet_kernel = [&] (int offset, int size) { + //auto verlet_kernel = [&i, &id_i, &slice_id, &p1, &cell_list, &cell_structure, &verlet_criterion, &distance_function, &verlet_list] (int offset, int size) { + for (int j = offset; j < offset + size; j++) { + // int jj = j; + int jj = cell_list.permutation(j); + int id_j = slice_id(jj); + if (id_i < id_j) { + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) + continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(i, jj); + /*std::cout << "*Cabana* " + << i << " " + << jj << " " + << id_i << " " + << id_j << " " + << slice_position(i, 0) << ", " + << slice_position(i, 1) << ", " + << slice_position(i, 2) << " " + << slice_position(jj, 0) << ", " + << slice_position(jj, 1) << ", " + << slice_position(jj, 2) << "\n";*/ + /*std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + } + } + } + }; + + verlet_kernel(cell_offset, cell_size); + + //Lees-Edwards BC + /*if (le_crossing != 0 && index[le_direction] == 1) { + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + cell_num[le_direction]) % cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - 1 + cell_num[le_direction]) % cell_num[le_direction]; + } + cell_offset = bin_offset(dx[0], dx[1], dx[2]); + cell_size = bin_size(dx[0], dx[1], dx[2]); + + verlet_kernel(cell_offset, cell_size); + }*/ } }; @@ -558,18 +599,6 @@ void cabana_short_range( local_torque(thread_id, j, 1) += opf.torque[1]; local_torque(thread_id, j, 2) += opf.torque[2]; - /*if (p1->id() == 512 || p2->id() == 512) { - std::cout << "2 CHECK FORCE i " - << i << " " << j << " " - << p1->id() << " " - << p2->id() << " " - << dist << " " - << p1->is_ghost() << " " - << p2->is_ghost() << " " - << pf.f[0] << " " - << pf.f[1] << " " - << pf.f[2] << "\n"; - }*/ #ifdef NPT local_virial(thread_id, 0) += virial[0]; local_virial(thread_id, 1) += virial[1]; @@ -606,18 +635,14 @@ void cabana_short_range( coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, num_threads, rank); - // std::cout << rank << " " << num_threads << " Execute - // FirstNeighborKernel\n"; - // TODO: Add option to switch "SerialOpTag" Between "TeamOpTag" - // Feels like TeamOpTag is faster, atleast for large particle numbers Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), - Cabana::SerialOpTag()); //, "verlet_list"); + Cabana::SerialOpTag()); Kokkos::fence(); // Force and Torque reduction Kokkos::parallel_for( - "reduce", policy, KOKKOS_LAMBDA(const int i) { + "reduction", policy, KOKKOS_LAMBDA(const int i) { double fx = 0.; double fy = 0.; double fz = 0.; @@ -638,14 +663,6 @@ void cabana_short_range( slice_torque(i, 0) = tx; slice_torque(i, 1) = ty; slice_torque(i, 2) = tz; - - /*if (slice_id(i) == 325 || slice_id(i) == 512) { - std::cout << "3 CHECK FORCE " - << slice_id(i) << " " - << slice_force(i, 0) << " " - << slice_force(i, 1) << " " - << slice_force(i, 2) << "\n"; - }*/ }); Kokkos::fence(); From c168deff7eb5771c9c36ee16f37bbeb8b8016399 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 11 Jun 2025 23:44:52 +0200 Subject: [PATCH 13/94] Formatting --- src/core/custom_verlet_list.hpp | 5 +- src/core/short_range_cabana.hpp | 156 ++++++++++++++++++-------------- 2 files changed, 89 insertions(+), 72 deletions(-) diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index df9919f9236..611d83a000c 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -86,8 +86,9 @@ class CustomVerletList void addNeighbor(const int pid, const int nid) { std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); if (count >= neighbors.extent(1)) { - //expandMaxNeighbors(neighbors.extent(1) * 2); - throw std::runtime_error("Number of count is larger than VerletList size."); + // expandMaxNeighbors(neighbors.extent(1) * 2); + throw std::runtime_error( + "Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 1cab9386fc4..19bc4e5ef43 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -163,15 +163,15 @@ void cabana_short_range( for (auto const &p : particles) { id_to_index[p.id()] = index; - //index_to_id.emplace_back(p.id()); + // index_to_id.emplace_back(p.id()); index++; } for (auto const &p : ghost_particles) { if (not id_to_index.contains(p.id())) { id_to_index[p.id()] = index; - //if (not contains(index_to_id, p.id())) { - //index_to_id.emplace_back(p.id()); + // if (not contains(index_to_id, p.id())) { + // index_to_id.emplace_back(p.id()); index++; } } @@ -256,10 +256,13 @@ void cabana_short_range( // Rebuild verlet list if needed auto const &system = ::System::get_system(); double max_cutoff = system.get_interaction_range(); - int max_counts = static_cast(27*max_cutoff*max_cutoff*max_cutoff/3); - if (max_counts < 64) max_counts = 64; + int max_counts = + static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); + if (max_counts < 64) + max_counts = 64; if (rebuild) { - verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); + verlet_list = + ListType(slice_position, 0, slice_position.size(), max_counts); /*auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); @@ -294,7 +297,7 @@ void cabana_short_range( for (int d = 0; d < 3; ++d) { eff_cutoff = max_cutoff; if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; + eff_cutoff = box_l[d]; cell_num[d] = static_cast(box_l[d] / eff_cutoff); grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); } @@ -313,14 +316,16 @@ void cabana_short_range( le_direction = box_geo.lees_edwards_bc().shear_direction; le_normal = box_geo.lees_edwards_bc().shear_plane_normal; delta_lebc[le_direction] = - static_cast(std::ceil(le_offset/grid_delta[le_direction])) % cell_num[le_direction]; + static_cast(std::ceil(le_offset / grid_delta[le_direction])) % + cell_num[le_direction]; } cell_list = Cabana::createLinkedCellList( slice_position, grid_delta, grid_min, grid_max); // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. // Cabana::permute( cell_list, particle_storage ); if (rebuild && max_cutoff != INACTIVE_CUTOFF) { - verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); + verlet_list = + ListType(slice_position, 0, slice_position.size(), max_counts); for (int cid = 0; cid < cell_list.totalBins(); ++cid) { cell_list(cid); } @@ -352,8 +357,11 @@ void cabana_short_range( std::as_const(cell_structure).decomposition().box()}; auto kernel = [&](const int i) { - //auto kernel = [&slice_id, &slice_ghost, &cell_structure, &particle_bins, &cell_list, &ijkIndexes, &cell_num, - // &le_protocol, &le_direction, &le_normal, &delta_lebc, &bin_offset, &bin_size, &verlet_criterion, &distance_function, &verlet_list](const int i) { + // auto kernel = [&slice_id, &slice_ghost, &cell_structure, + // &particle_bins, &cell_list, &ijkIndexes, &cell_num, + // &le_protocol, &le_direction, &le_normal, &delta_lebc, &bin_offset, + // &bin_size, &verlet_criterion, &distance_function, + // &verlet_list](const int i) { int id_i = slice_id(i); if (slice_ghost(i)) return; @@ -364,78 +372,86 @@ void cabana_short_range( cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); int dx[3]; for (int n = 0; n < 27; ++n) { - bool duplicate_cell = false; + bool duplicate_cell = false; for (int d = 0; d < 3; ++d) { dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; - if (cell_num[d] <= 2 && ijkIndexes[n][d] + index[d] != dx[d]) - duplicate_cell = true; + if (cell_num[d] <= 2 && ijkIndexes[n][d] + index[d] != dx[d]) + duplicate_cell = true; + } + if (duplicate_cell) + continue; + + // Lees-Edwards BC + int le_crossing = 0; + if (le_protocol != nullptr) { + le_crossing = + ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } } - if (duplicate_cell) continue; - - //Lees-Edwards BC - int le_crossing = 0; - if (le_protocol != nullptr) { - le_crossing = - ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + cell_num[le_direction]) % cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + cell_num[le_direction]) % cell_num[le_direction]; - } - } int cell_offset = bin_offset(dx[0], dx[1], dx[2]); int cell_size = bin_size(dx[0], dx[1], dx[2]); - auto verlet_kernel = [&] (int offset, int size) { - //auto verlet_kernel = [&i, &id_i, &slice_id, &p1, &cell_list, &cell_structure, &verlet_criterion, &distance_function, &verlet_list] (int offset, int size) { - for (int j = offset; j < offset + size; j++) { - // int jj = j; - int jj = cell_list.permutation(j); - int id_j = slice_id(jj); - if (id_i < id_j) { - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) - continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(i, jj); - /*std::cout << "*Cabana* " - << i << " " - << jj << " " - << id_i << " " - << id_j << " " - << slice_position(i, 0) << ", " - << slice_position(i, 1) << ", " - << slice_position(i, 2) << " " - << slice_position(jj, 0) << ", " - << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";*/ - /*std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ - } - } - } + auto verlet_kernel = [&](int offset, int size) { + // auto verlet_kernel = [&i, &id_i, &slice_id, &p1, &cell_list, + // &cell_structure, &verlet_criterion, &distance_function, + // &verlet_list] (int offset, int size) { + for (int j = offset; j < offset + size; j++) { + // int jj = j; + int jj = cell_list.permutation(j); + int id_j = slice_id(jj); + if (id_i < id_j) { + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) + continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(i, jj); + /*std::cout << "*Cabana* " + << i << " " + << jj << " " + << id_i << " " + << id_j << " " + << slice_position(i, 0) << ", " + << slice_position(i, 1) << ", " + << slice_position(i, 2) << " " + << slice_position(jj, 0) << ", " + << slice_position(jj, 1) << ", " + << slice_position(jj, 2) << "\n";*/ + /*std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + } + } + } }; - verlet_kernel(cell_offset, cell_size); + verlet_kernel(cell_offset, cell_size); - //Lees-Edwards BC - /*if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + cell_num[le_direction]) % cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - 1 + cell_num[le_direction]) % cell_num[le_direction]; - } + // Lees-Edwards BC + /*if (le_crossing != 0 && index[le_direction] == 1) { + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + + cell_num[le_direction]) % cell_num[le_direction]; } else if + (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } cell_offset = bin_offset(dx[0], dx[1], dx[2]); cell_size = bin_size(dx[0], dx[1], dx[2]); - verlet_kernel(cell_offset, cell_size); - }*/ + verlet_kernel(cell_offset, cell_size); + }*/ } }; From 91b9531d9f08f64539aca491eae4ca4d9da0dac0 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 12:05:36 +0200 Subject: [PATCH 14/94] Modified core/CMakeLists.txt due to warning from cabana and kokkos --- src/core/CMakeLists.txt | 5 +++++ src/core/short_range_cabana.hpp | 27 ++++++++++++++------------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index da836dcf977..8835fe6ac1a 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -101,6 +101,11 @@ target_link_libraries( Boost::serialization Boost::mpi espresso::instrumentation) target_include_directories(espresso_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) + target_include_directories(espresso_core SYSTEM PUBLIC + ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src + ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) +endif() if(ESPRESSO_BUILD_WITH_WALBERLA) target_link_libraries( diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 19bc4e5ef43..7407687586a 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -145,6 +145,7 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - Index map"); #endif std::unordered_map id_to_index{}; // For DEBUG + std::unordered_set registered_index{}; std::vector index_to_id{}; int index = 0; @@ -162,14 +163,17 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - id_to_index[p.id()] = index; + //id_to_index[p.id()] = index; + registered_index.insert(p.id()); // index_to_id.emplace_back(p.id()); index++; } for (auto const &p : ghost_particles) { - if (not id_to_index.contains(p.id())) { - id_to_index[p.id()] = index; + if (not registered_index.contains(p.id())) { + registered_index.insert(p.id()); + //if (not id_to_index.contains(p.id())) { + //id_to_index[p.id()] = index; // if (not contains(index_to_id, p.id())) { // index_to_id.emplace_back(p.id()); index++; @@ -179,7 +183,7 @@ void cabana_short_range( // If we do not rebuild we can use the saved map id_to_index = saved_data.get_id_to_index(); index_to_id = saved_data.get_index_to_id(); - index = id_to_index.size(); + index = registered_index.size(); } const int number_of_unique_particles = index; @@ -204,24 +208,24 @@ void cabana_short_range( auto slice_ghost = Cabana::slice<6>(particle_storage); auto box_l = box_geo.length(); int p_id = 0; - std::vector registered_pid{}; + registered_index.clear(); for (auto const &p : particles) { write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); if (p.is_ghost()) std::cout << "WIRED!!!!!!!!!\n"; - registered_pid.emplace_back(p.id()); + registered_index.insert(p.id()); ++p_id; } for (auto const &p : ghost_particles) { // if the ghost is not in the previous map, but mpi moved it to this rank? // it will not have neighbors because we did not rebuild the verlet list. - if (contains(registered_pid, p.id())) { + if (registered_index.contains(p.id())) { continue; } write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); - registered_pid.emplace_back(p.id()); + registered_index.insert(p.id()); ++p_id; } @@ -356,12 +360,9 @@ void cabana_short_range( auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; + // This kernel will be changed to the loop for the pair of intaracted cell id. + // Now, per 1 cell, 27 neighbor cell is calculated and it is wasteful. auto kernel = [&](const int i) { - // auto kernel = [&slice_id, &slice_ghost, &cell_structure, - // &particle_bins, &cell_list, &ijkIndexes, &cell_num, - // &le_protocol, &le_direction, &le_normal, &delta_lebc, &bin_offset, - // &bin_size, &verlet_criterion, &distance_function, - // &verlet_list](const int i) { int id_i = slice_id(i); if (slice_ghost(i)) return; From b96a9f541600223ae560843027ed4c04be00c517 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 12:06:53 +0200 Subject: [PATCH 15/94] Formatting --- src/core/CMakeLists.txt | 6 +++--- src/core/short_range_cabana.hpp | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 8835fe6ac1a..333342a848b 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -102,9 +102,9 @@ target_link_libraries( target_include_directories(espresso_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - target_include_directories(espresso_core SYSTEM PUBLIC - ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src - ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) + target_include_directories( + espresso_core SYSTEM PUBLIC ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src + ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) endif() if(ESPRESSO_BUILD_WITH_WALBERLA) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 7407687586a..12423653930 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -163,19 +163,19 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - //id_to_index[p.id()] = index; - registered_index.insert(p.id()); + // id_to_index[p.id()] = index; + registered_index.insert(p.id()); // index_to_id.emplace_back(p.id()); index++; } for (auto const &p : ghost_particles) { - if (not registered_index.contains(p.id())) { - registered_index.insert(p.id()); - //if (not id_to_index.contains(p.id())) { - //id_to_index[p.id()] = index; - // if (not contains(index_to_id, p.id())) { - // index_to_id.emplace_back(p.id()); + if (not registered_index.contains(p.id())) { + registered_index.insert(p.id()); + // if (not id_to_index.contains(p.id())) { + // id_to_index[p.id()] = index; + // if (not contains(index_to_id, p.id())) { + // index_to_id.emplace_back(p.id()); index++; } } @@ -360,8 +360,8 @@ void cabana_short_range( auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; - // This kernel will be changed to the loop for the pair of intaracted cell id. - // Now, per 1 cell, 27 neighbor cell is calculated and it is wasteful. + // This kernel will be changed to the loop for the pair of intaracted cell + // id. Now, per 1 cell, 27 neighbor cell is calculated and it is wasteful. auto kernel = [&](const int i) { int id_i = slice_id(i); if (slice_ghost(i)) From 80e8e254bc1e9418ca9f3c55103c13f64468d60d Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 12:19:06 +0200 Subject: [PATCH 16/94] Added comments --- src/core/CMakeLists.txt | 5 +++-- src/core/short_range_cabana.hpp | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 333342a848b..1c696af8e4f 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -103,8 +103,9 @@ target_link_libraries( target_include_directories(espresso_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) target_include_directories( - espresso_core SYSTEM PUBLIC ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src - ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) + espresso_core SYSTEM INTERFACE + ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src + ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) endif() if(ESPRESSO_BUILD_WITH_WALBERLA) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 12423653930..8d76951a931 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -335,6 +335,7 @@ void cabana_short_range( } auto const particle_bins = cell_list.getParticleBins(); + // Offset particle id and the number of particle in specific cell Kokkos::View bin_offset( "bin_offset", cell_num[0], cell_num[1], cell_num[2]); Kokkos::View bin_size( @@ -349,6 +350,8 @@ void cabana_short_range( cell_list.binOffset(dx[0], dx[1], dx[2]); bin_size(dx[0], dx[1], dx[2]) = cell_list.binSize(dx[0], dx[1], dx[2]); } + + // Interacting cell constexpr int ijkIndexes[27][3] = { {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, @@ -402,9 +405,6 @@ void cabana_short_range( int cell_size = bin_size(dx[0], dx[1], dx[2]); auto verlet_kernel = [&](int offset, int size) { - // auto verlet_kernel = [&i, &id_i, &slice_id, &p1, &cell_list, - // &cell_structure, &verlet_criterion, &distance_function, - // &verlet_list] (int offset, int size) { for (int j = offset; j < offset + size; j++) { // int jj = j; int jj = cell_list.permutation(j); From fc5e46d0e71bd3acc06d539c64ec32b1facd9cd6 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 12:21:43 +0200 Subject: [PATCH 17/94] Formatting --- src/core/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1c696af8e4f..9fe2c4e6057 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -103,9 +103,8 @@ target_link_libraries( target_include_directories(espresso_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) target_include_directories( - espresso_core SYSTEM INTERFACE - ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src - ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) + espresso_core SYSTEM INTERFACE ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src + ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) endif() if(ESPRESSO_BUILD_WITH_WALBERLA) From ddd553cda19f200decb7d4191ac381c0faf717a4 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 19:46:36 +0200 Subject: [PATCH 18/94] Improved computation rate --- src/core/short_range_cabana.hpp | 216 +++++++++++++++++-------- testsuite/python/scafacos_interface.py | 4 +- 2 files changed, 148 insertions(+), 72 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 8d76951a931..141cc935f42 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -163,7 +163,7 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - // id_to_index[p.id()] = index; + //id_to_index[p.id()] = index; registered_index.insert(p.id()); // index_to_id.emplace_back(p.id()); index++; @@ -173,7 +173,7 @@ void cabana_short_range( if (not registered_index.contains(p.id())) { registered_index.insert(p.id()); // if (not id_to_index.contains(p.id())) { - // id_to_index[p.id()] = index; + //id_to_index[p.id()] = index; // if (not contains(index_to_id, p.id())) { // index_to_id.emplace_back(p.id()); index++; @@ -259,24 +259,31 @@ void cabana_short_range( // Rebuild verlet list if needed auto const &system = ::System::get_system(); + int max_counts; double max_cutoff = system.get_interaction_range(); - int max_counts = + if (std::isinf(max_cutoff)) { + max_counts = number_of_unique_particles; + } else { + max_counts = static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); + } if (max_counts < 64) max_counts = 64; if (rebuild) { - verlet_list = + /*verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); - /*auto kernel = [&](Particle const &p1, Particle const &p2) { + auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); - //std::cout << "Cell_structure " + std::cout << "Cell_structure " << id_to_index.at(p1.id()) << " " << id_to_index.at(p2.id()) << " " + << p1.is_ghost() << " " + << p2.is_ghost() << " " << p1.id() << " " << p2.id() << " " << p1.pos() << " " - << p2.pos() << "\n";// + << p2.pos() << "\n"; //if (p1.id() < p2.id()) { // pair_check.emplace_back(std::pair{p1.id(), p2.id()}); //} else { @@ -325,33 +332,36 @@ void cabana_short_range( } cell_list = Cabana::createLinkedCellList( slice_position, grid_delta, grid_min, grid_max); + int total_bins = cell_list.totalBins(); // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. // Cabana::permute( cell_list, particle_storage ); if (rebuild && max_cutoff != INACTIVE_CUTOFF) { verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); - for (int cid = 0; cid < cell_list.totalBins(); ++cid) { + for (int cid = 0; cid < total_bins; ++cid) { cell_list(cid); } auto const particle_bins = cell_list.getParticleBins(); // Offset particle id and the number of particle in specific cell - Kokkos::View bin_offset( - "bin_offset", cell_num[0], cell_num[1], cell_num[2]); - Kokkos::View bin_size( - "bin_size", cell_num[0], cell_num[1], cell_num[2]); - for (int cid = 0; cid < cell_list.totalBins(); ++cid) { + Kokkos::View bin_offset( + "bin_offset", total_bins); + Kokkos::View bin_size( + "bin_size", total_bins); + for (int cid = 0; cid < total_bins; ++cid) { int dx[3] = {}; dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / cell_num[2]); dx[2] = cid % cell_num[2]; - bin_offset(dx[0], dx[1], dx[2]) = + bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); - bin_size(dx[0], dx[1], dx[2]) = cell_list.binSize(dx[0], dx[1], dx[2]); + bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); + //int cardinal_id = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); + //std::cout << "COMPARE " << cid << " " << cardinal_id << "\n"; } - // Interacting cell + // Creating Interacting cell constexpr int ijkIndexes[27][3] = { {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, @@ -359,21 +369,19 @@ void cabana_short_range( {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, {1, 1, 0}, {1, 1, 1}}; - - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - - // This kernel will be changed to the loop for the pair of intaracted cell - // id. Now, per 1 cell, 27 neighbor cell is calculated and it is wasteful. - auto kernel = [&](const int i) { - int id_i = slice_id(i); - if (slice_ghost(i)) - return; - auto p1 = cell_structure.get_local_particle(id_i); - if (p1 == nullptr) - return; + int total_pair_cell; + if (total_bins < 27) { + total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; + } else { + total_pair_cell = 14 * total_bins; + } + //std::cout << "TotalBins=" << total_bins << "\n"; + //std::cout << "TotalPairCell=" << total_pair_cell << "\n"; + Kokkos::View interacting_pair_cell("interacting_pair_cell", total_pair_cell, 2); + int pair_cell_id = 0; + for (int cid_i = 0; cid_i < total_bins; ++cid_i) { int index[3] = {}; - cell_list.ijkBinIndex(particle_bins(i), index[0], index[1], index[2]); + cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); int dx[3]; for (int n = 0; n < 27; ++n) { bool duplicate_cell = false; @@ -397,48 +405,111 @@ void cabana_short_range( } else if (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + cell_num[le_direction]) % - cell_num[le_direction]; + cell_num[le_direction]; } } - int cell_offset = bin_offset(dx[0], dx[1], dx[2]); - int cell_size = bin_size(dx[0], dx[1], dx[2]); - - auto verlet_kernel = [&](int offset, int size) { - for (int j = offset; j < offset + size; j++) { - // int jj = j; - int jj = cell_list.permutation(j); - int id_j = slice_id(jj); - if (id_i < id_j) { - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) - continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(i, jj); - /*std::cout << "*Cabana* " - << i << " " - << jj << " " - << id_i << " " - << id_j << " " - << slice_position(i, 0) << ", " - << slice_position(i, 1) << ", " - << slice_position(i, 2) << " " - << slice_position(jj, 0) << ", " - << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";*/ - /*std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ - } - } - } - }; + int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); + if (cid_i <= cid_j) { + if (pair_cell_id >= total_pair_cell) { + std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " << pair_cell_id << "\n"; + std::terminate(); // or handle safely + } + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } + } + } + // End Creating Interacting cell - verlet_kernel(cell_offset, cell_size); + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; + + // This kernel used the loop for the pair of interacting cell + auto kernel = [&](const int pair_cell_i) { + + int cid_i = interacting_pair_cell(pair_cell_i, 0); + int cid_j = interacting_pair_cell(pair_cell_i, 1); + + auto verlet_kernel = [&](Particle *p1, int i, int id_i, int cell_offset, int cell_size) { + for (int j = cell_offset; j < cell_offset + cell_size; ++j) { + // int jj = j; + int ii = cell_list.permutation(i); //debug + int jj = cell_list.permutation(j); + int id_j = slice_id(jj); + if (slice_ghost(ii) && slice_ghost(jj)) continue; // reject both ghost + + if (slice_ghost(ii) || slice_ghost(jj)) { + if (id_i < id_j && slice_ghost(ii)) { + continue; + } else if (id_i > id_j && slice_ghost(jj)) { + continue; + } + } + /*if (cid_i == cid_j) { + if (id_i < id_j && slice_ghost(ii)) { + continue; + } else if (id_i > id_j && slice_ghost(jj)) { + continue; + } + }*/ + if (1) { + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) + continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << slice_ghost(ii) << " " + << slice_ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << slice_position(ii, 0) << ", " + << slice_position(ii, 1) << ", " + << slice_position(ii, 2) << " " + << slice_position(jj, 0) << ", " + << slice_position(jj, 1) << ", " + << slice_position(jj, 2) << "\n";*/ + /*std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + } + } + } // j-loop + }; + + int offset_i = bin_offset(cid_i); + int size_i = bin_size(cid_i); + + for (int i = offset_i; i < offset_i + size_i; ++i) { + // int ii = i; + int ii = cell_list.permutation(i); + int id_i = slice_id(ii); + //if (slice_ghost(ii)) + // continue; + auto p1 = cell_structure.get_local_particle(id_i); + if (p1 == nullptr) + continue; + + if (cid_i == cid_j) { + //verlet_kernel(p1, ii, id_i, i + 1, size_i + offset_i - i - 1); // j-loop + verlet_kernel(p1, i, id_i, i + 1, size_i + offset_i - i - 1); // j-loop + } else { + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); + //verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop + verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop + } + } // i-loop // Lees-Edwards BC /*if (le_crossing != 0 && index[le_direction] == 1) { @@ -453,12 +524,17 @@ void cabana_short_range( verlet_kernel(cell_offset, cell_size); }*/ - } }; - Kokkos::RangePolicy policy(0, particle_storage.size()); + Kokkos::RangePolicy policy(0, total_pair_cell); Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); + /*for (int pair_i = 0; pair_i < total_pair_cell; ++pair_i) { + std::cout << "CHECK " << total_pair_cell << " " + << pair_i << " " + << interacting_pair_cell(pair_i, 0) << " " + << interacting_pair_cell(pair_i, 1) << "\n"; + }*/ } // Save data for next iteration if we just rebuilt diff --git a/testsuite/python/scafacos_interface.py b/testsuite/python/scafacos_interface.py index 5c0776a7d00..992a9fc0e55 100644 --- a/testsuite/python/scafacos_interface.py +++ b/testsuite/python/scafacos_interface.py @@ -350,8 +350,8 @@ def fcs_data(self): new_torques = np.copy(system.part.all().torque_lab) self.assertAlmostEqual(new_E_coulomb, ref_E_coulomb, delta=0) self.assertAlmostEqual(new_E_dipoles, ref_E_dipoles, delta=0) - np.testing.assert_allclose(new_forces, ref_forces, atol=0, rtol=0.) - np.testing.assert_allclose(new_torques, ref_torques, atol=0, rtol=0.) + np.testing.assert_allclose(new_forces, ref_forces, atol=1e-10, rtol=1e-10) + np.testing.assert_allclose(new_torques, ref_torques, atol=1e-10, rtol=1e-10) self.system.electrostatics.clear() self.system.magnetostatics.clear() From 835c2365dbae97a275f04d113ede82bcd7728eca Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 19:48:40 +0200 Subject: [PATCH 19/94] Formatting --- src/core/short_range_cabana.hpp | 259 +++++++++++++------------ testsuite/python/scafacos_interface.py | 6 +- 2 files changed, 135 insertions(+), 130 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 141cc935f42..06194f05036 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -163,7 +163,7 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - //id_to_index[p.id()] = index; + // id_to_index[p.id()] = index; registered_index.insert(p.id()); // index_to_id.emplace_back(p.id()); index++; @@ -173,7 +173,7 @@ void cabana_short_range( if (not registered_index.contains(p.id())) { registered_index.insert(p.id()); // if (not id_to_index.contains(p.id())) { - //id_to_index[p.id()] = index; + // id_to_index[p.id()] = index; // if (not contains(index_to_id, p.id())) { // index_to_id.emplace_back(p.id()); index++; @@ -265,7 +265,7 @@ void cabana_short_range( max_counts = number_of_unique_particles; } else { max_counts = - static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); + static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); } if (max_counts < 64) max_counts = 64; @@ -278,8 +278,8 @@ void cabana_short_range( std::cout << "Cell_structure " << id_to_index.at(p1.id()) << " " << id_to_index.at(p2.id()) << " " - << p1.is_ghost() << " " - << p2.is_ghost() << " " + << p1.is_ghost() << " " + << p2.is_ghost() << " " << p1.id() << " " << p2.id() << " " << p1.pos() << " " @@ -344,21 +344,19 @@ void cabana_short_range( auto const particle_bins = cell_list.getParticleBins(); // Offset particle id and the number of particle in specific cell - Kokkos::View bin_offset( - "bin_offset", total_bins); - Kokkos::View bin_size( - "bin_size", total_bins); + Kokkos::View bin_offset("bin_offset", + total_bins); + Kokkos::View bin_size("bin_size", total_bins); for (int cid = 0; cid < total_bins; ++cid) { int dx[3] = {}; dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / cell_num[2]); dx[2] = cid % cell_num[2]; - bin_offset(cid) = - cell_list.binOffset(dx[0], dx[1], dx[2]); + bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); - //int cardinal_id = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - //std::cout << "COMPARE " << cid << " " << cardinal_id << "\n"; + // int cardinal_id = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); + // std::cout << "COMPARE " << cid << " " << cardinal_id << "\n"; } // Creating Interacting cell @@ -371,13 +369,14 @@ void cabana_short_range( {1, 1, 0}, {1, 1, 1}}; int total_pair_cell; if (total_bins < 27) { - total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; + total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; } else { - total_pair_cell = 14 * total_bins; + total_pair_cell = 14 * total_bins; } - //std::cout << "TotalBins=" << total_bins << "\n"; - //std::cout << "TotalPairCell=" << total_pair_cell << "\n"; - Kokkos::View interacting_pair_cell("interacting_pair_cell", total_pair_cell, 2); + // std::cout << "TotalBins=" << total_bins << "\n"; + // std::cout << "TotalPairCell=" << total_pair_cell << "\n"; + Kokkos::View interacting_pair_cell( + "interacting_pair_cell", total_pair_cell, 2); int pair_cell_id = 0; for (int cid_i = 0; cid_i < total_bins; ++cid_i) { int index[3] = {}; @@ -405,21 +404,22 @@ void cabana_short_range( } else if (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + cell_num[le_direction]) % - cell_num[le_direction]; + cell_num[le_direction]; } } - int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - if (cid_i <= cid_j) { - if (pair_cell_id >= total_pair_cell) { - std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " << pair_cell_id << "\n"; - std::terminate(); // or handle safely - } - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - ++pair_cell_id; - } - } + int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); + if (cid_i <= cid_j) { + if (pair_cell_id >= total_pair_cell) { + std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " + << pair_cell_id << "\n"; + std::terminate(); // or handle safely + } + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } + } } // End Creating Interacting cell @@ -428,112 +428,115 @@ void cabana_short_range( // This kernel used the loop for the pair of interacting cell auto kernel = [&](const int pair_cell_i) { - - int cid_i = interacting_pair_cell(pair_cell_i, 0); - int cid_j = interacting_pair_cell(pair_cell_i, 1); - - auto verlet_kernel = [&](Particle *p1, int i, int id_i, int cell_offset, int cell_size) { - for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - // int jj = j; - int ii = cell_list.permutation(i); //debug - int jj = cell_list.permutation(j); - int id_j = slice_id(jj); - if (slice_ghost(ii) && slice_ghost(jj)) continue; // reject both ghost - - if (slice_ghost(ii) || slice_ghost(jj)) { - if (id_i < id_j && slice_ghost(ii)) { - continue; - } else if (id_i > id_j && slice_ghost(jj)) { - continue; - } - } - /*if (cid_i == cid_j) { - if (id_i < id_j && slice_ghost(ii)) { - continue; - } else if (id_i > id_j && slice_ghost(jj)) { - continue; - } - }*/ - if (1) { - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) - continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " - << slice_ghost(ii) << " " - << slice_ghost(jj) << " " - << cid_i << " " - << cid_j << " " - << slice_position(ii, 0) << ", " - << slice_position(ii, 1) << ", " - << slice_position(ii, 2) << " " - << slice_position(jj, 0) << ", " - << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";*/ - /*std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ - } - } - } // j-loop - }; - - int offset_i = bin_offset(cid_i); - int size_i = bin_size(cid_i); - - for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; - int ii = cell_list.permutation(i); - int id_i = slice_id(ii); - //if (slice_ghost(ii)) - // continue; - auto p1 = cell_structure.get_local_particle(id_i); - if (p1 == nullptr) - continue; - - if (cid_i == cid_j) { - //verlet_kernel(p1, ii, id_i, i + 1, size_i + offset_i - i - 1); // j-loop - verlet_kernel(p1, i, id_i, i + 1, size_i + offset_i - i - 1); // j-loop - } else { - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - //verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop - verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop - } - } // i-loop - - // Lees-Edwards BC - /*if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + - cell_num[le_direction]) % cell_num[le_direction]; } else if - (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - 1 + - cell_num[le_direction]) % cell_num[le_direction]; + int cid_i = interacting_pair_cell(pair_cell_i, 0); + int cid_j = interacting_pair_cell(pair_cell_i, 1); + + auto verlet_kernel = [&](Particle *p1, int i, int id_i, int cell_offset, + int cell_size) { + for (int j = cell_offset; j < cell_offset + cell_size; ++j) { + // int jj = j; + int ii = cell_list.permutation(i); // debug + int jj = cell_list.permutation(j); + int id_j = slice_id(jj); + if (slice_ghost(ii) && slice_ghost(jj)) + continue; // reject both ghost + + if (slice_ghost(ii) || slice_ghost(jj)) { + if (id_i < id_j && slice_ghost(ii)) { + continue; + } else if (id_i > id_j && slice_ghost(jj)) { + continue; + } + } + /*if (cid_i == cid_j) { + if (id_i < id_j && slice_ghost(ii)) { + continue; + } else if (id_i > id_j && slice_ghost(jj)) { + continue; + } + }*/ + if (1) { + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) + continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << slice_ghost(ii) << " " + << slice_ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << slice_position(ii, 0) << ", " + << slice_position(ii, 1) << ", " + << slice_position(ii, 2) << " " + << slice_position(jj, 0) << ", " + << slice_position(jj, 1) << ", " + << slice_position(jj, 2) << "\n";*/ + /*std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + } } - cell_offset = bin_offset(dx[0], dx[1], dx[2]); - cell_size = bin_size(dx[0], dx[1], dx[2]); + } // j-loop + }; + + int offset_i = bin_offset(cid_i); + int size_i = bin_size(cid_i); + + for (int i = offset_i; i < offset_i + size_i; ++i) { + // int ii = i; + int ii = cell_list.permutation(i); + int id_i = slice_id(ii); + // if (slice_ghost(ii)) + // continue; + auto p1 = cell_structure.get_local_particle(id_i); + if (p1 == nullptr) + continue; + + if (cid_i == cid_j) { + // verlet_kernel(p1, ii, id_i, i + 1, size_i + offset_i - i - 1); // + // j-loop + verlet_kernel(p1, i, id_i, i + 1, + size_i + offset_i - i - 1); // j-loop + } else { + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); + // verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop + verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop + } + } // i-loop + + // Lees-Edwards BC + /*if (le_crossing != 0 && index[le_direction] == 1) { + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + + cell_num[le_direction]) % cell_num[le_direction]; } else if + (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } + cell_offset = bin_offset(dx[0], dx[1], dx[2]); + cell_size = bin_size(dx[0], dx[1], dx[2]); - verlet_kernel(cell_offset, cell_size); - }*/ + verlet_kernel(cell_offset, cell_size); + }*/ }; Kokkos::RangePolicy policy(0, total_pair_cell); Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); /*for (int pair_i = 0; pair_i < total_pair_cell; ++pair_i) { - std::cout << "CHECK " << total_pair_cell << " " - << pair_i << " " - << interacting_pair_cell(pair_i, 0) << " " - << interacting_pair_cell(pair_i, 1) << "\n"; + std::cout << "CHECK " << total_pair_cell << " " + << pair_i << " " + << interacting_pair_cell(pair_i, 0) << " " + << interacting_pair_cell(pair_i, 1) << "\n"; }*/ } diff --git a/testsuite/python/scafacos_interface.py b/testsuite/python/scafacos_interface.py index 992a9fc0e55..6977a8a0579 100644 --- a/testsuite/python/scafacos_interface.py +++ b/testsuite/python/scafacos_interface.py @@ -350,8 +350,10 @@ def fcs_data(self): new_torques = np.copy(system.part.all().torque_lab) self.assertAlmostEqual(new_E_coulomb, ref_E_coulomb, delta=0) self.assertAlmostEqual(new_E_dipoles, ref_E_dipoles, delta=0) - np.testing.assert_allclose(new_forces, ref_forces, atol=1e-10, rtol=1e-10) - np.testing.assert_allclose(new_torques, ref_torques, atol=1e-10, rtol=1e-10) + np.testing.assert_allclose( + new_forces, ref_forces, atol=1e-10, rtol=1e-10) + np.testing.assert_allclose( + new_torques, ref_torques, atol=1e-10, rtol=1e-10) self.system.electrostatics.clear() self.system.magnetostatics.clear() From c466a176f3295afd1b0743f092c74d625d318795 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 23:50:56 +0200 Subject: [PATCH 20/94] Removed empty cell from calculations --- src/core/short_range_cabana.hpp | 89 +++++++++++++++++++-------------- testsuite/python/exclusions.py | 3 ++ 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 06194f05036..f750281a7c6 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -72,9 +72,9 @@ inline void write_particle(Particle const &p, int const &id, s_torque(id, 0) = 0.0; s_torque(id, 1) = 0.0; s_torque(id, 2) = 0.0; - assert(s_position(id, 0) >= 0. && s_position(id, 0) < box_l[0]); - assert(s_position(id, 1) >= 0. && s_position(id, 1) < box_l[1]); - assert(s_position(id, 2) >= 0. && s_position(id, 2) < box_l[2]); + assert(s_position(id, 0) >= 0. and s_position(id, 0) < box_l[0]); + assert(s_position(id, 1) >= 0. and s_position(id, 1) < box_l[1]); + assert(s_position(id, 2) >= 0. and s_position(id, 2) < box_l[2]); } template @@ -253,7 +253,7 @@ void cabana_short_range( // Get Verlet Pairs and Fill list // =================================================== #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List"); + CALI_MARK_BEGIN("Cabana - Verlet List1"); #endif ListType verlet_list; @@ -335,13 +335,10 @@ void cabana_short_range( int total_bins = cell_list.totalBins(); // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. // Cabana::permute( cell_list, particle_storage ); - if (rebuild && max_cutoff != INACTIVE_CUTOFF) { + if (rebuild and max_cutoff != INACTIVE_CUTOFF) { + verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); - for (int cid = 0; cid < total_bins; ++cid) { - cell_list(cid); - } - auto const particle_bins = cell_list.getParticleBins(); // Offset particle id and the number of particle in specific cell Kokkos::View bin_offset("bin_offset", @@ -355,9 +352,11 @@ void cabana_short_range( dx[2] = cid % cell_num[2]; bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); - // int cardinal_id = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - // std::cout << "COMPARE " << cid << " " << cardinal_id << "\n"; + + // Calculate particle_bins + cell_list(cid); } + auto const particle_bins = cell_list.getParticleBins(); // Creating Interacting cell constexpr int ijkIndexes[27][3] = { @@ -375,18 +374,25 @@ void cabana_short_range( } // std::cout << "TotalBins=" << total_bins << "\n"; // std::cout << "TotalPairCell=" << total_pair_cell << "\n"; + /* + * Creating list of interacting pair cell + */ Kokkos::View interacting_pair_cell( "interacting_pair_cell", total_pair_cell, 2); + int empty_pair_number = 0; int pair_cell_id = 0; for (int cid_i = 0; cid_i < total_bins; ++cid_i) { + // Obtaining 3 dimentional cell index from cid_i int index[3] = {}; cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); int dx[3]; + // From 27 neighbor cell, the list of interacting pair cell is created for (int n = 0; n < 27; ++n) { bool duplicate_cell = false; + // Obtaining 3 dimentional cell index from neighbor cell for (int d = 0; d < 3; ++d) { dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; - if (cell_num[d] <= 2 && ijkIndexes[n][d] + index[d] != dx[d]) + if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) duplicate_cell = true; } if (duplicate_cell) @@ -408,16 +414,22 @@ void cabana_short_range( } } + + // Interacting pair cell is registered in the list int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); if (cid_i <= cid_j) { - if (pair_cell_id >= total_pair_cell) { - std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " - << pair_cell_id << "\n"; - std::terminate(); // or handle safely - } - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - ++pair_cell_id; + if (bin_size(cid_i) != 0 and bin_size(cid_j) !=0) { + if (pair_cell_id >= total_pair_cell) { + std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " + << pair_cell_id << "\n"; + std::terminate(); // or handle safely + } + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } else { + ++empty_pair_number; + } } } } @@ -425,7 +437,12 @@ void cabana_short_range( auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; - +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List1"); +#endif +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Verlet List2"); +#endif // This kernel used the loop for the pair of interacting cell auto kernel = [&](const int pair_cell_i) { int cid_i = interacting_pair_cell(pair_cell_i, 0); @@ -438,15 +455,17 @@ void cabana_short_range( int ii = cell_list.permutation(i); // debug int jj = cell_list.permutation(j); int id_j = slice_id(jj); - if (slice_ghost(ii) && slice_ghost(jj)) + if (slice_ghost(ii) and slice_ghost(jj)) { continue; // reject both ghost - - if (slice_ghost(ii) || slice_ghost(jj)) { - if (id_i < id_j && slice_ghost(ii)) { - continue; - } else if (id_i > id_j && slice_ghost(jj)) { - continue; - } + } + if (slice_ghost(ii) or slice_ghost(jj)) { + //if (cid_i == cid_j) { + if (id_i < id_j and slice_ghost(ii)) { + continue; + } else if (id_i > id_j and slice_ghost(jj)) { + continue; + } + //} } /*if (cid_i == cid_j) { if (id_i < id_j && slice_ghost(ii)) { @@ -529,15 +548,9 @@ void cabana_short_range( }*/ }; - Kokkos::RangePolicy policy(0, total_pair_cell); + Kokkos::RangePolicy policy(0, total_pair_cell - empty_pair_number); Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); - /*for (int pair_i = 0; pair_i < total_pair_cell; ++pair_i) { - std::cout << "CHECK " << total_pair_cell << " " - << pair_i << " " - << interacting_pair_cell(pair_i, 0) << " " - << interacting_pair_cell(pair_i, 1) << "\n"; - }*/ } // Save data for next iteration if we just rebuilt @@ -678,7 +691,7 @@ void cabana_short_range( virial, d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); -#endif +#endif //ETC // local_force(thread_id, i, 0) += pf.f[0]; local_force(thread_id, i, 1) += pf.f[1]; @@ -709,7 +722,7 @@ void cabana_short_range( }; }; #ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List"); + CALI_MARK_END("Cabana - Verlet List2"); #endif // =================================================== diff --git a/testsuite/python/exclusions.py b/testsuite/python/exclusions.py index 4cc1aaccb5f..17887ee208d 100644 --- a/testsuite/python/exclusions.py +++ b/testsuite/python/exclusions.py @@ -57,7 +57,10 @@ def test_transfer(self): p0.exclusions = [1, 2, 3] + i = 0 for _ in range(15): + print(i) + i += 1 self.system.integrator.run(100) self.assertEqual(list(p0.exclusions), [1, 2, 3]) From 2085860e65fc32dc923ce169698167eb68be33ed Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 12 Jun 2025 23:52:26 +0200 Subject: [PATCH 21/94] Formatting --- src/core/short_range_cabana.hpp | 60 ++++++++++++++++----------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index f750281a7c6..9900d3d597b 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -353,7 +353,7 @@ void cabana_short_range( bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); - // Calculate particle_bins + // Calculate particle_bins cell_list(cid); } auto const particle_bins = cell_list.getParticleBins(); @@ -382,14 +382,14 @@ void cabana_short_range( int empty_pair_number = 0; int pair_cell_id = 0; for (int cid_i = 0; cid_i < total_bins; ++cid_i) { - // Obtaining 3 dimentional cell index from cid_i + // Obtaining 3 dimentional cell index from cid_i int index[3] = {}; cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); int dx[3]; - // From 27 neighbor cell, the list of interacting pair cell is created + // From 27 neighbor cell, the list of interacting pair cell is created for (int n = 0; n < 27; ++n) { bool duplicate_cell = false; - // Obtaining 3 dimentional cell index from neighbor cell + // Obtaining 3 dimentional cell index from neighbor cell for (int d = 0; d < 3; ++d) { dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) @@ -414,22 +414,21 @@ void cabana_short_range( } } - // Interacting pair cell is registered in the list int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); if (cid_i <= cid_j) { - if (bin_size(cid_i) != 0 and bin_size(cid_j) !=0) { - if (pair_cell_id >= total_pair_cell) { - std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " - << pair_cell_id << "\n"; - std::terminate(); // or handle safely - } - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - ++pair_cell_id; - } else { - ++empty_pair_number; - } + if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { + if (pair_cell_id >= total_pair_cell) { + std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " + << pair_cell_id << "\n"; + std::terminate(); // or handle safely + } + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } else { + ++empty_pair_number; + } } } } @@ -438,10 +437,10 @@ void cabana_short_range( auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; #ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List1"); + CALI_MARK_END("Cabana - Verlet List1"); #endif #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List2"); + CALI_MARK_BEGIN("Cabana - Verlet List2"); #endif // This kernel used the loop for the pair of interacting cell auto kernel = [&](const int pair_cell_i) { @@ -457,15 +456,15 @@ void cabana_short_range( int id_j = slice_id(jj); if (slice_ghost(ii) and slice_ghost(jj)) { continue; // reject both ghost - } + } if (slice_ghost(ii) or slice_ghost(jj)) { - //if (cid_i == cid_j) { - if (id_i < id_j and slice_ghost(ii)) { - continue; - } else if (id_i > id_j and slice_ghost(jj)) { - continue; - } - //} + // if (cid_i == cid_j) { + if (id_i < id_j and slice_ghost(ii)) { + continue; + } else if (id_i > id_j and slice_ghost(jj)) { + continue; + } + //} } /*if (cid_i == cid_j) { if (id_i < id_j && slice_ghost(ii)) { @@ -548,7 +547,8 @@ void cabana_short_range( }*/ }; - Kokkos::RangePolicy policy(0, total_pair_cell - empty_pair_number); + Kokkos::RangePolicy policy(0, total_pair_cell - + empty_pair_number); Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); } @@ -691,8 +691,8 @@ void cabana_short_range( virial, d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); -#endif //ETC - // +#endif // ETC + // local_force(thread_id, i, 0) += pf.f[0]; local_force(thread_id, i, 1) += pf.f[1]; local_force(thread_id, i, 2) += pf.f[2]; From 61f84f01f321ab0d49945c8290ddddb719ecda78 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Sat, 14 Jun 2025 18:10:34 +0200 Subject: [PATCH 22/94] Modified rebuild_flag --- src/core/cabana_data.hpp | 11 +-- src/core/cell_system/CellStructure.cpp | 22 ++++-- src/core/cell_system/CellStructure.hpp | 18 ++++- src/core/short_range_cabana.hpp | 105 ++++++++++++++----------- src/core/system/System.cpp | 1 + 5 files changed, 97 insertions(+), 60 deletions(-) diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index a9b427d0887..c14b73d7aa0 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -26,7 +26,8 @@ #include #include -using data_types = Cabana::MemberTypes; +using data_types = Cabana::MemberTypes; using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -35,6 +36,7 @@ using ListType = Cabana::CustomVerletList; class CabanaData { + Cabana::AoSoA particle_storage; ListType verlet_list; std::unordered_map id_to_index; std::vector index_to_id; @@ -43,10 +45,9 @@ class CabanaData { CabanaData() = default; CabanaData(ListType verlet_list, std::unordered_map id_to_index) : verlet_list(verlet_list), id_to_index(id_to_index) {} - CabanaData(ListType verlet_list, std::unordered_map id_to_index, - std::vector index_to_id) - : verlet_list(verlet_list), id_to_index(id_to_index), - index_to_id(index_to_id) {} + CabanaData(Cabana::AoSoA &particle_storage, + ListType &verlet_list, std::unordered_map &id_to_index) + : particle_storage(particle_storage), verlet_list(verlet_list), id_to_index(id_to_index) {} ListType get_verlet_list() const { return verlet_list; } std::unordered_map get_id_to_index() const { return id_to_index; } diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index d8b3a404fd2..1b5f61722ef 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -60,7 +60,8 @@ #ifdef SHARED_MEMORY_PARALLELISM -using data_types = Cabana::MemberTypes; +using data_types = Cabana::MemberTypes; using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -68,20 +69,28 @@ using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; -CellStructure::~CellStructure() { m_cabana_data.reset(); } +CellStructure::~CellStructure() { + if (m_cabana_data) { + m_cabana_data.reset(); + } +} void CellStructure::set_cabana_data(std::unique_ptr data) { m_cabana_data = std::move(data); - // m_rebuild_verlet_list = false; - // m_rebuild_cabana_verlet_list = false; + m_rebuild_verlet_list = false; + //std::cout << "c1.rebuild " << m_rebuild_verlet_list << std::endl; + m_rebuild_cabana_verlet_list = false; } CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } void CellStructure::reset_cabana_data() { - m_rebuild_verlet_list = true; + //m_rebuild_verlet_list = true; + //std::cout << "c2.rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = true; - m_cabana_data.reset(); + if (m_cabana_data) { + m_cabana_data.reset(); + } } #endif @@ -265,6 +274,7 @@ void CellStructure::resort_particles(bool global_flag) { auto const &lebc = get_system().box_geo->lees_edwards_bc(); m_rebuild_verlet_list = true; + //std::cout << "resort-rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = true; m_le_pos_offset_at_last_resort = lebc.pos_offset; diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index be7f8adf34c..79fb3b15e39 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -55,6 +55,7 @@ #include #include #include +#include // forward declaration to not have to import cabana #ifdef SHARED_MEMORY_PARALLELISM @@ -687,19 +688,25 @@ struct CellStructure : public System::Leaf { template void cabana_verlet_list_loop(Kernel kernel, const VerletCriterion &verlet_criterion) { - if (m_rebuild_cabana_verlet_list) { + //if (m_rebuild_cabana_verlet_list) { + if (m_rebuild_verlet_list) { m_verlet_list.clear(); link_cell([&](Particle &p1, Particle &p2, Distance const &d) { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); + //std::cout << "WITHOUT CS " + // << p1.id() << " " + // << p2.id() << std::endl; } }); - m_rebuild_cabana_verlet_list = false; + m_rebuild_verlet_list = false; } for (auto const &pair : m_verlet_list) { kernel(*pair.first, *pair.second); } + m_rebuild_cabana_verlet_list = false; + //std::cout << "h1.rebuild " << m_rebuild_verlet_list << std::endl; } #endif @@ -715,6 +722,7 @@ struct CellStructure : public System::Leaf { /* In this case the verlet list update is attached to * the pair kernel, and the verlet list is rebuilt as * we go. */ + //std::cout << "In verlet_list_looop " << m_rebuild_verlet_list << " " << m_rebuild_cabana_verlet_list << std::endl; if (m_rebuild_verlet_list) { m_verlet_list.clear(); @@ -722,10 +730,15 @@ struct CellStructure : public System::Leaf { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); pair_kernel(p1, p2, d); + //std::cout << "WITHOUT CS " + // << p1.id() << " " + // << p2.id() << std::endl; } }); m_rebuild_verlet_list = false; + //std::cout << "h2.rebuild " << m_rebuild_verlet_list << std::endl; + m_rebuild_cabana_verlet_list = true; } else { auto const maybe_box = decomposition().minimum_image_distance(); /* In this case the pair kernel is just run over the verlet list. */ @@ -771,6 +784,7 @@ struct CellStructure : public System::Leaf { template void non_bonded_loop(PairKernel pair_kernel, const VerletCriterion &verlet_criterion) { + std::cout << "non_bonded_loop " << use_verlet_list << std::endl; if (use_verlet_list) { verlet_list_loop(pair_kernel, verlet_criterion); } else { diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 9900d3d597b..394a7a96cd3 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -150,20 +150,17 @@ void cabana_short_range( int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); + //std::cout << "For CABANA rebuild " << rebuild << std::endl; CabanaData saved_data; - // Load saved data if we do not have to rebuild - if (!rebuild) { - saved_data = cell_structure.get_cabana_data(); - } // If we have to rebuild, we need to count the particles and create a new // map if (rebuild) { for (auto const &p : particles) { - // id_to_index[p.id()] = index; + id_to_index[p.id()] = index; registered_index.insert(p.id()); // index_to_id.emplace_back(p.id()); index++; @@ -172,14 +169,13 @@ void cabana_short_range( for (auto const &p : ghost_particles) { if (not registered_index.contains(p.id())) { registered_index.insert(p.id()); - // if (not id_to_index.contains(p.id())) { - // id_to_index[p.id()] = index; - // if (not contains(index_to_id, p.id())) { - // index_to_id.emplace_back(p.id()); + id_to_index[p.id()] = index; index++; } } } else { + // Load saved data if we do not have to rebuild + saved_data = cell_structure.get_cabana_data(); // If we do not rebuild we can use the saved map id_to_index = saved_data.get_id_to_index(); index_to_id = saved_data.get_index_to_id(); @@ -212,8 +208,6 @@ void cabana_short_range( for (auto const &p : particles) { write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); - if (p.is_ghost()) - std::cout << "WIRED!!!!!!!!!\n"; registered_index.insert(p.id()); ++p_id; } @@ -248,7 +242,7 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_END("Cabana - Fill particle storage"); #endif - + // START VERLET_LIST // =================================================== // Get Verlet Pairs and Fill list // =================================================== @@ -270,20 +264,20 @@ void cabana_short_range( if (max_counts < 64) max_counts = 64; if (rebuild) { - /*verlet_list = + verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); - std::cout << "Cell_structure " - << id_to_index.at(p1.id()) << " " - << id_to_index.at(p2.id()) << " " - << p1.is_ghost() << " " - << p2.is_ghost() << " " - << p1.id() << " " - << p2.id() << " " - << p1.pos() << " " - << p2.pos() << "\n"; + //std::cout << "WITHSMP " + //<< id_to_index.at(p1.id()) << " " + //<< id_to_index.at(p2.id()) << " " + //<< p1.is_ghost() << " " + //<< p2.is_ghost() << " " + //<< p1.id() << " " + //<< p2.id() << std::endl; + //<< p1.pos() << " " + //<< p2.pos() << "\n"; //if (p1.id() < p2.id()) { // pair_check.emplace_back(std::pair{p1.id(), p2.id()}); //} else { @@ -291,12 +285,15 @@ void cabana_short_range( //} }; - cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);*/ + cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);// } else { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); } - +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List1"); +#endif + /* // Creating LinkedCellList and VerletList: // Box Properties Cabana::LinkedCellList cell_list; @@ -374,9 +371,9 @@ void cabana_short_range( } // std::cout << "TotalBins=" << total_bins << "\n"; // std::cout << "TotalPairCell=" << total_pair_cell << "\n"; - /* - * Creating list of interacting pair cell - */ + // + // Creating list of interacting pair cell + // Kokkos::View interacting_pair_cell( "interacting_pair_cell", total_pair_cell, 2); int empty_pair_number = 0; @@ -466,20 +463,13 @@ void cabana_short_range( } //} } - /*if (cid_i == cid_j) { - if (id_i < id_j && slice_ghost(ii)) { - continue; - } else if (id_i > id_j && slice_ghost(jj)) { - continue; - } - }*/ if (1) { auto p2 = cell_structure.get_local_particle(id_j); if (p2 == nullptr) continue; if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { verlet_list.addNeighbor(ii, jj); - /*std::cout << "*Cabana* " + //std::cout << "*Cabana* " << i << " " << j << " " << id_i << " " @@ -493,14 +483,14 @@ void cabana_short_range( << slice_position(ii, 2) << " " << slice_position(jj, 0) << ", " << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";*/ - /*std::cout << "CHECK " + << slice_position(jj, 2) << "\n";// + //std::cout << "CHECK " << n << " " << i << " " << j << " " << dx[0] << " " << dx[1] << " " - << dx[2] << "\n";*/ + << dx[2] << "\n";// } } } // j-loop @@ -533,7 +523,7 @@ void cabana_short_range( } // i-loop // Lees-Edwards BC - /*if (le_crossing != 0 && index[le_direction] == 1) { + //if (le_crossing != 0 && index[le_direction] == 1) { if (le_crossing < 0) { dx[le_direction] = (dx[le_direction] + 1 + cell_num[le_direction]) % cell_num[le_direction]; } else if @@ -544,7 +534,7 @@ void cabana_short_range( cell_size = bin_size(dx[0], dx[1], dx[2]); verlet_kernel(cell_offset, cell_size); - }*/ + }// }; Kokkos::RangePolicy policy(0, total_pair_cell - @@ -552,10 +542,12 @@ void cabana_short_range( Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); } + */ //END VERLET_LIST // Save data for next iteration if we just rebuilt if (rebuild) { - CabanaData new_data(verlet_list, id_to_index); + CabanaData new_data(particle_storage, verlet_list, id_to_index); + //CabanaData new_data(verlet_list, id_to_index); cell_structure.set_cabana_data(std::make_unique(new_data)); } @@ -586,6 +578,7 @@ void cabana_short_range( int num_threads; int mpi_rank; + int particle_number; FirstNeighborKernel( const CellStructure *cell_, @@ -608,7 +601,7 @@ void cabana_short_range( Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, - int num_threads_, int mpi_rank_) + int num_threads_, int mpi_rank_, int particle_number_) : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), thermostat(thermostat_), box_geo(box_geo_), // index_to_id(index_to_id_), @@ -621,12 +614,17 @@ void cabana_short_range( #endif coulomb_kernel(coulomb_kernel_), dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), - num_threads(num_threads_), mpi_rank(mpi_rank_) { + num_threads(num_threads_), mpi_rank(mpi_rank_), particle_number(particle_number_) { } KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { + if (i >= particle_number or j >= particle_number) { + std::cerr << "ERROR: index exceeds number_of_unique_particles " + << i << " " << j << "\n"; + std::terminate(); // or handle safely + } Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), slice_position(i, 2)}; Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), @@ -721,9 +719,9 @@ void cabana_short_range( #endif }; }; -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List2"); -#endif +//#ifdef CALIPER +// CALI_MARK_END("Cabana - Verlet List2"); +//#endif // =================================================== // Execute Kernel @@ -742,7 +740,7 @@ void cabana_short_range( *collision_detection, #endif coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, - num_threads, rank); + num_threads, rank, number_of_unique_particles); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), @@ -752,6 +750,11 @@ void cabana_short_range( // Force and Torque reduction Kokkos::parallel_for( "reduction", policy, KOKKOS_LAMBDA(const int i) { + if (i >= number_of_unique_particles) { + std::cerr << "ERROR: index exceeds number_of_unique_particles " + << i << " " << "\n"; + std::terminate(); // or handle safely + } double fx = 0.; double fy = 0.; double fz = 0.; @@ -801,6 +804,9 @@ void cabana_short_range( collision_detection->detect_collision(p1, p2, d.dist2); } }; + //bool const rebuild_e = cell_structure.get_rebuild_verlet_list(); + //bool const rebuild_c = cell_structure.get_rebuild_cabana_verlet_list(); + //std::cout << "Both should be 0 before non_bonded_loop " << rebuild_e << " " << rebuild_c << std::endl; cell_structure.non_bonded_loop(collision_kernel, verlet_criterion); #endif #ifdef CALIPER @@ -818,6 +824,11 @@ void cabana_short_range( if (p == nullptr) { return; } + if (id >= number_of_unique_particles) { + std::cerr << "ERROR: id exceeds number_of_unique_particles " + << id << " " << "\n"; + std::terminate(); // or handle safely + } Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), slice_force(id, 2)}; Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index cb5d7851da1..fce28161761 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -95,6 +95,7 @@ System::System(Private) { } System::~System() { + //std::cout << "~System()\n"; #ifdef SHARED_MEMORY_PARALLELISM cell_structure->reset_cabana_data(); #endif From e6b9f7e9f39f1cb1840ad95a74306e8eb1b24847 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Sun, 15 Jun 2025 16:27:31 +0200 Subject: [PATCH 23/94] Small refactoring and Fixed rebuild_flag --- src/core/cabana_data.hpp | 12 +- src/core/cell_system/CellStructure.cpp | 8 +- src/core/cell_system/CellStructure.hpp | 4 +- src/core/forces.cpp | 2 +- src/core/short_range_cabana.hpp | 724 ++++++++++++------------- 5 files changed, 359 insertions(+), 391 deletions(-) diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index c14b73d7aa0..34b0f5df2fa 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -26,8 +26,6 @@ #include #include -using data_types = Cabana::MemberTypes; using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -36,22 +34,24 @@ using ListType = Cabana::CustomVerletList; class CabanaData { - Cabana::AoSoA particle_storage; ListType verlet_list; std::unordered_map id_to_index; std::vector index_to_id; + int particle_number; public: CabanaData() = default; + CabanaData(ListType verlet_list, std::unordered_map id_to_index, std::vector index_to_id) + : verlet_list(verlet_list), id_to_index(id_to_index), index_to_id(index_to_id) {} CabanaData(ListType verlet_list, std::unordered_map id_to_index) : verlet_list(verlet_list), id_to_index(id_to_index) {} - CabanaData(Cabana::AoSoA &particle_storage, - ListType &verlet_list, std::unordered_map &id_to_index) - : particle_storage(particle_storage), verlet_list(verlet_list), id_to_index(id_to_index) {} + CabanaData(ListType verlet_list, int particle_number) + : verlet_list(verlet_list), particle_number(particle_number) {} ListType get_verlet_list() const { return verlet_list; } std::unordered_map get_id_to_index() const { return id_to_index; } std::vector get_index_to_id() const { return index_to_id; } + int get_index() const { return particle_number; } ~CabanaData() {}; }; diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 1b5f61722ef..c81792f4693 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -60,8 +60,6 @@ #ifdef SHARED_MEMORY_PARALLELISM -using data_types = Cabana::MemberTypes; using memory_space = Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -77,7 +75,7 @@ CellStructure::~CellStructure() { void CellStructure::set_cabana_data(std::unique_ptr data) { m_cabana_data = std::move(data); - m_rebuild_verlet_list = false; + //m_rebuild_verlet_list = false; //std::cout << "c1.rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = false; } @@ -85,9 +83,9 @@ void CellStructure::set_cabana_data(std::unique_ptr data) { CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } void CellStructure::reset_cabana_data() { - //m_rebuild_verlet_list = true; + m_rebuild_verlet_list = true; //std::cout << "c2.rebuild " << m_rebuild_verlet_list << std::endl; - m_rebuild_cabana_verlet_list = true; + //m_rebuild_cabana_verlet_list = true; if (m_cabana_data) { m_cabana_data.reset(); } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 79fb3b15e39..983b5c776ae 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -701,12 +701,12 @@ struct CellStructure : public System::Leaf { } }); m_rebuild_verlet_list = false; + //m_rebuild_cabana_verlet_list = false; } for (auto const &pair : m_verlet_list) { kernel(*pair.first, *pair.second); } m_rebuild_cabana_verlet_list = false; - //std::cout << "h1.rebuild " << m_rebuild_verlet_list << std::endl; } #endif @@ -784,7 +784,7 @@ struct CellStructure : public System::Leaf { template void non_bonded_loop(PairKernel pair_kernel, const VerletCriterion &verlet_criterion) { - std::cout << "non_bonded_loop " << use_verlet_list << std::endl; + //std::cout << "non_bonded_loop " << use_verlet_list << std::endl; if (use_verlet_list) { verlet_list_loop(pair_kernel, verlet_criterion); } else { diff --git a/src/core/forces.cpp b/src/core/forces.cpp index e40e37ef25d..1624e2a6a94 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -203,7 +203,7 @@ void System::System::calculate_forces() { #ifdef COLLISION_DETECTION collision_detection, #endif - *cell_structure, maximal_cutoff(), bonded_ias->maximal_cutoff(), + *cell_structure, get_interaction_range(), bonded_ias->maximal_cutoff(), *thermostat, *box_geo, *nonbonded_ias, particles, cell_structure->ghost_particles(), VerletCriterion<>{*this, cell_structure->get_verlet_skin(), diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 394a7a96cd3..726ae2e1e1c 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -36,9 +36,9 @@ #include #include #include -#include #include #include +#include inline double wrap(double x, double L) { auto result = x - std::floor(x / L) * L; @@ -47,10 +47,6 @@ inline double wrap(double x, double L) { return result; } -inline bool contains(std::vector const &storage, int const value) { - return (std::find(storage.begin(), storage.end(), value) != storage.end()); -} - template inline void write_particle(Particle const &p, int const &id, @@ -77,6 +73,100 @@ inline void write_particle(Particle const &p, int const &id, assert(s_position(id, 2) >= 0. and s_position(id, 2) < box_l[2]); } +inline void set_offset_and_size_indexed_by_cid( int &total_bins, int* cell_num, + Cabana::LinkedCellList &cell_list, + Kokkos::View &bin_offset, + Kokkos::View &bin_size) { + for (int cid = 0; cid < total_bins; ++cid) { + int dx[3] = {}; + dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); + dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / + cell_num[2]); + dx[2] = cid % cell_num[2]; + bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); + bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); + + // Calculate particle_bins + cell_list(cid); + } +} +using ActiveProtocol = std::variant; +inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int* cell_num, + int* delta_lebc, int le_direction, int le_normal, + std::shared_ptr le_protocol, + Kokkos::View &bin_size, + Cabana::LinkedCellList &cell_list, + Kokkos::View &interacting_pair_cell) { + + constexpr int ijkIndexes[27][3] = { + {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, + {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, + {0, -1, 0}, {0, -1, 1}, {0, 0, -1}, {0, 0, 0}, {0, 0, 1}, + {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, + {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, + {1, 1, 0}, {1, 1, 1}}; + // std::cout << "TotalBins=" << total_bins << "\n"; + // std::cout << "TotalPairCell=" << total_pair_cell << "\n"; + // + // Creating list of interacting pair cell + // + int empty_pair_number = 0; + int pair_cell_id = 0; + for (int cid_i = 0; cid_i < total_bins; ++cid_i) { + // Obtaining 3 dimentional cell index from cid_i + int index[3] = {}; + cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); + int dx[3]; + // From 27 neighbor cell, the list of interacting pair cell is created + for (int n = 0; n < 27; ++n) { + bool duplicate_cell = false; + // Obtaining 3 dimentional cell index from neighbor cell + for (int d = 0; d < 3; ++d) { + dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; + if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) + duplicate_cell = true; + } + if (duplicate_cell) + continue; + + // Lees-Edwards BC + int le_crossing = 0; + if (le_protocol != nullptr) { + le_crossing = + ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } + } + + // Interacting pair cell is registered in the list + int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); + if (cid_i <= cid_j) { + if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { + /*if (pair_cell_id >= total_pair_cell) { + std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " + << pair_cell_id << "\n"; + std::terminate(); // or handle safely + }*/ + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } else { + ++empty_pair_number; + } + } + } + } + return empty_pair_number; +} + + template void cabana_short_range( BondKernel bond_kernel, @@ -100,6 +190,7 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Espresso - Bond Kernel"); #endif + assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); if (bond_cutoff >= 0.) { @@ -154,6 +245,10 @@ void cabana_short_range( CabanaData saved_data; + // Load saved data if we do not have to rebuild + if (!rebuild) { + saved_data = cell_structure.get_cabana_data(); + } // If we have to rebuild, we need to count the particles and create a new // map @@ -162,24 +257,20 @@ void cabana_short_range( for (auto const &p : particles) { id_to_index[p.id()] = index; registered_index.insert(p.id()); - // index_to_id.emplace_back(p.id()); index++; } for (auto const &p : ghost_particles) { if (not registered_index.contains(p.id())) { - registered_index.insert(p.id()); id_to_index[p.id()] = index; + registered_index.insert(p.id()); index++; } } } else { - // Load saved data if we do not have to rebuild - saved_data = cell_structure.get_cabana_data(); // If we do not rebuild we can use the saved map - id_to_index = saved_data.get_id_to_index(); - index_to_id = saved_data.get_index_to_id(); - index = registered_index.size(); + //id_to_index = saved_data.get_id_to_index(); + index = saved_data.get_index(); } const int number_of_unique_particles = index; @@ -242,316 +333,8 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_END("Cabana - Fill particle storage"); #endif - // START VERLET_LIST - // =================================================== - // Get Verlet Pairs and Fill list - // =================================================== -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List1"); -#endif - ListType verlet_list; - - // Rebuild verlet list if needed - auto const &system = ::System::get_system(); - int max_counts; - double max_cutoff = system.get_interaction_range(); - if (std::isinf(max_cutoff)) { - max_counts = number_of_unique_particles; - } else { - max_counts = - static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); - } - if (max_counts < 64) - max_counts = 64; - if (rebuild) { - verlet_list = - ListType(slice_position, 0, slice_position.size(), max_counts); - auto kernel = [&](Particle const &p1, Particle const &p2) { - verlet_list.addNeighbor(id_to_index.at(p1.id()), - id_to_index.at(p2.id())); - //std::cout << "WITHSMP " - //<< id_to_index.at(p1.id()) << " " - //<< id_to_index.at(p2.id()) << " " - //<< p1.is_ghost() << " " - //<< p2.is_ghost() << " " - //<< p1.id() << " " - //<< p2.id() << std::endl; - //<< p1.pos() << " " - //<< p2.pos() << "\n"; - //if (p1.id() < p2.id()) { - // pair_check.emplace_back(std::pair{p1.id(), p2.id()}); - //} else { - // pair_check.emplace_back(std::pair{p2.id(), p1.id()}); - //} - }; - - cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);// - } else { - // Else use the saved verlet list - verlet_list = saved_data.get_verlet_list(); - } -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List1"); -#endif - /* - // Creating LinkedCellList and VerletList: - // Box Properties - Cabana::LinkedCellList cell_list; - double grid_min[3] = {0.0, 0.0, 0.0}; - double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; - double grid_delta[3] = {}; - int cell_num[3] = {}; - double eff_cutoff; - for (int d = 0; d < 3; ++d) { - eff_cutoff = max_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); - } - // Lees-Edwards boundary condition - double le_offset; - int le_direction; - int le_normal; - int delta_lebc[3] = {0, 0, 0}; - auto le_protocol = system.lees_edwards->get_protocol(); - if (le_protocol == nullptr) { - le_offset = 0.; - le_direction = -1; - le_normal = -1; - } else { - le_offset = box_geo.lees_edwards_bc().pos_offset; - le_direction = box_geo.lees_edwards_bc().shear_direction; - le_normal = box_geo.lees_edwards_bc().shear_plane_normal; - delta_lebc[le_direction] = - static_cast(std::ceil(le_offset / grid_delta[le_direction])) % - cell_num[le_direction]; - } - cell_list = Cabana::createLinkedCellList( - slice_position, grid_delta, grid_min, grid_max); - int total_bins = cell_list.totalBins(); - // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. - // Cabana::permute( cell_list, particle_storage ); - if (rebuild and max_cutoff != INACTIVE_CUTOFF) { - - verlet_list = - ListType(slice_position, 0, slice_position.size(), max_counts); - - // Offset particle id and the number of particle in specific cell - Kokkos::View bin_offset("bin_offset", - total_bins); - Kokkos::View bin_size("bin_size", total_bins); - for (int cid = 0; cid < total_bins; ++cid) { - int dx[3] = {}; - dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); - dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / - cell_num[2]); - dx[2] = cid % cell_num[2]; - bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); - bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); - - // Calculate particle_bins - cell_list(cid); - } - auto const particle_bins = cell_list.getParticleBins(); - - // Creating Interacting cell - constexpr int ijkIndexes[27][3] = { - {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, - {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, - {0, -1, 0}, {0, -1, 1}, {0, 0, -1}, {0, 0, 0}, {0, 0, 1}, - {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, - {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, - {1, 1, 0}, {1, 1, 1}}; - int total_pair_cell; - if (total_bins < 27) { - total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; - } else { - total_pair_cell = 14 * total_bins; - } - // std::cout << "TotalBins=" << total_bins << "\n"; - // std::cout << "TotalPairCell=" << total_pair_cell << "\n"; - // - // Creating list of interacting pair cell - // - Kokkos::View interacting_pair_cell( - "interacting_pair_cell", total_pair_cell, 2); - int empty_pair_number = 0; - int pair_cell_id = 0; - for (int cid_i = 0; cid_i < total_bins; ++cid_i) { - // Obtaining 3 dimentional cell index from cid_i - int index[3] = {}; - cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); - int dx[3]; - // From 27 neighbor cell, the list of interacting pair cell is created - for (int n = 0; n < 27; ++n) { - bool duplicate_cell = false; - // Obtaining 3 dimentional cell index from neighbor cell - for (int d = 0; d < 3; ++d) { - dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; - if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) - duplicate_cell = true; - } - if (duplicate_cell) - continue; - - // Lees-Edwards BC - int le_crossing = 0; - if (le_protocol != nullptr) { - le_crossing = - ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } - } - - // Interacting pair cell is registered in the list - int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - if (cid_i <= cid_j) { - if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { - if (pair_cell_id >= total_pair_cell) { - std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " - << pair_cell_id << "\n"; - std::terminate(); // or handle safely - } - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - ++pair_cell_id; - } else { - ++empty_pair_number; - } - } - } - } - // End Creating Interacting cell - - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List1"); -#endif -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List2"); -#endif - // This kernel used the loop for the pair of interacting cell - auto kernel = [&](const int pair_cell_i) { - int cid_i = interacting_pair_cell(pair_cell_i, 0); - int cid_j = interacting_pair_cell(pair_cell_i, 1); - - auto verlet_kernel = [&](Particle *p1, int i, int id_i, int cell_offset, - int cell_size) { - for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - // int jj = j; - int ii = cell_list.permutation(i); // debug - int jj = cell_list.permutation(j); - int id_j = slice_id(jj); - if (slice_ghost(ii) and slice_ghost(jj)) { - continue; // reject both ghost - } - if (slice_ghost(ii) or slice_ghost(jj)) { - // if (cid_i == cid_j) { - if (id_i < id_j and slice_ghost(ii)) { - continue; - } else if (id_i > id_j and slice_ghost(jj)) { - continue; - } - //} - } - if (1) { - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) - continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); - //std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " - << slice_ghost(ii) << " " - << slice_ghost(jj) << " " - << cid_i << " " - << cid_j << " " - << slice_position(ii, 0) << ", " - << slice_position(ii, 1) << ", " - << slice_position(ii, 2) << " " - << slice_position(jj, 0) << ", " - << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";// - //std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";// - } - } - } // j-loop - }; - - int offset_i = bin_offset(cid_i); - int size_i = bin_size(cid_i); - - for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; - int ii = cell_list.permutation(i); - int id_i = slice_id(ii); - // if (slice_ghost(ii)) - // continue; - auto p1 = cell_structure.get_local_particle(id_i); - if (p1 == nullptr) - continue; - - if (cid_i == cid_j) { - // verlet_kernel(p1, ii, id_i, i + 1, size_i + offset_i - i - 1); // - // j-loop - verlet_kernel(p1, i, id_i, i + 1, - size_i + offset_i - i - 1); // j-loop - } else { - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - // verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop - verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop - } - } // i-loop - - // Lees-Edwards BC - //if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + - cell_num[le_direction]) % cell_num[le_direction]; } else if - (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } - cell_offset = bin_offset(dx[0], dx[1], dx[2]); - cell_size = bin_size(dx[0], dx[1], dx[2]); - - verlet_kernel(cell_offset, cell_size); - }// - }; - - Kokkos::RangePolicy policy(0, total_pair_cell - - empty_pair_number); - Kokkos::parallel_for("calc_by_cell_list", policy, kernel); - Kokkos::fence(); - } - */ //END VERLET_LIST - - // Save data for next iteration if we just rebuilt - if (rebuild) { - CabanaData new_data(particle_storage, verlet_list, id_to_index); - //CabanaData new_data(verlet_list, id_to_index); - cell_structure.set_cabana_data(std::make_unique(new_data)); - } - // calculate force with customverletlist + // The kernel of calculate force struct FirstNeighborKernel { const CellStructure *cell; [[maybe_unused]] const BondedInteractionsMap &bonded_ias; @@ -620,11 +403,6 @@ void cabana_short_range( KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { - if (i >= particle_number or j >= particle_number) { - std::cerr << "ERROR: index exceeds number_of_unique_particles " - << i << " " << j << "\n"; - std::terminate(); // or handle safely - } Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), slice_position(i, 2)}; Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), @@ -637,24 +415,24 @@ void cabana_short_range( auto thread_id = omp_get_thread_num(); // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - // std::cout << "in " << thread_id << " " << i << " " << j << " " << - // q1q2 << "\n"; + //std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << " " << IA_parameters const &ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); - /* + // auto p1 = cell->get_local_particle(slice_id(i)); auto p2 = cell->get_local_particle(slice_id(j)); if (p1 == nullptr or p2 == nullptr) return; + auto const dist2 = dist * dist; auto[pf, virial] = add_non_bonded_pair_force( const_cast(*p1), const_cast(*p2), d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); - */ // + /* ParticleForce pf{}; Utils::Vector3d virial{}; @@ -690,7 +468,7 @@ void cabana_short_range( thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); #endif // ETC - // + */ local_force(thread_id, i, 0) += pf.f[0]; local_force(thread_id, i, 1) += pf.f[1]; local_force(thread_id, i, 2) += pf.f[2]; @@ -719,17 +497,56 @@ void cabana_short_range( #endif }; }; -//#ifdef CALIPER -// CALI_MARK_END("Cabana - Verlet List2"); -//#endif + // START VERLET_LIST // =================================================== - // Execute Kernel + // Get Verlet Pairs and Fill list // =================================================== #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Execute Kernel"); + CALI_MARK_BEGIN("Cabana - Verlet List1"); +#endif + ListType verlet_list; + + // Rebuild verlet list if needed + auto const &system = ::System::get_system(); + int max_counts; + double max_cutoff = pair_cutoff; //system.get_interaction_range(); + if (std::isinf(max_cutoff)) { + max_counts = number_of_unique_particles; + } else { + max_counts = + static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); + } + if (max_counts < 128) + max_counts = 128; + if (rebuild) { // Legacy Velert List + verlet_list = + ListType(slice_position, 0, slice_position.size(), max_counts); + auto kernel = [&](Particle const &p1, Particle const &p2) { + verlet_list.addNeighbor(id_to_index.at(p1.id()), + id_to_index.at(p2.id())); + //std::cout << "WITHSMP " + //<< id_to_index.at(p1.id()) << " " + //<< id_to_index.at(p2.id()) << " " + //<< p1.is_ghost() << " " + //<< p2.is_ghost() << " " + //<< p1.id() << " " + //<< p2.id() << std::endl; + //<< p1.pos() << " " + //<< p2.pos() << "\n"; + }; + + cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);// + } else { + // Else use the saved verlet list + verlet_list = saved_data.get_verlet_list(); + } +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List1"); +#endif +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Verlet List2"); #endif - Kokkos::RangePolicy policy(0, particle_storage.size()); FirstNeighborKernel first_neighbor_kernel( &cell_structure, bonded_ias, nonbonded_ias, thermostat, box_geo, @@ -742,12 +559,200 @@ void cabana_short_range( coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, num_threads, rank, number_of_unique_particles); - Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, - Cabana::FirstNeighborsTag(), - Cabana::SerialOpTag()); - Kokkos::fence(); + if (rebuild and max_cutoff != INACTIVE_CUTOFF and 0) { // Shared memory + // Creating LinkedCellList and VerletList: + // Box Properties + Cabana::LinkedCellList cell_list; + double grid_min[3] = {0.0, 0.0, 0.0}; + double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; + double grid_delta[3] = {}; + int cell_num[3] = {}; + double eff_cutoff; + for (int d = 0; d < 3; ++d) { + eff_cutoff = max_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + } + // For Lees-Edwards boundary condition + double le_offset; + int le_direction; + int le_normal; + int delta_lebc[3] = {0, 0, 0}; + auto le_protocol = system.lees_edwards->get_protocol(); + if (le_protocol == nullptr) { + le_offset = 0.; + le_direction = -1; + le_normal = -1; + } else { + le_offset = box_geo.lees_edwards_bc().pos_offset; + le_direction = box_geo.lees_edwards_bc().shear_direction; + le_normal = box_geo.lees_edwards_bc().shear_plane_normal; + delta_lebc[le_direction] = + static_cast(std::ceil(le_offset / grid_delta[le_direction])) % + cell_num[le_direction]; + } + cell_list = Cabana::createLinkedCellList( + slice_position, grid_delta, grid_min, grid_max); + int total_bins = cell_list.totalBins(); + // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. + // Cabana::permute( cell_list, particle_storage ); + + verlet_list = + ListType(slice_position, 0, slice_position.size(), max_counts); + + // Offset particle id and the number of particle in specific cell + Kokkos::View bin_offset("bin_offset", + total_bins); + Kokkos::View bin_size("bin_size", total_bins); + set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, bin_offset, bin_size); + auto const particle_bins = cell_list.getParticleBins(); + + + // Creating Interacting cell + int total_pair_cell; + if (total_bins < 27) { + total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; + } else { + total_pair_cell = 14 * total_bins; + } + Kokkos::View interacting_pair_cell( + "interacting_pair_cell", total_pair_cell, 2); + int empty_pair_number = + set_interacting_pair_cell(total_bins, total_pair_cell, cell_num, + delta_lebc, le_direction, le_normal, le_protocol, + bin_size, cell_list, interacting_pair_cell); + // End Creating Interacting cell + + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; + + // This kernel used the loop for the pair of interacting cell + auto kernel = [&](const int pair_cell_i) { + int cid_i = interacting_pair_cell(pair_cell_i, 0); + int cid_j = interacting_pair_cell(pair_cell_i, 1); + + auto verlet_kernel = [&](Particle *p1, int ii, int id_i, int cell_offset, + int cell_size) { + for (int j = cell_offset; j < cell_offset + cell_size; ++j) { + // int jj = j; + //int ii = cell_list.permutation(i); // debug + int jj = cell_list.permutation(j); + int id_j = slice_id(jj); + if (slice_ghost(ii) and slice_ghost(jj)) { + continue; // reject both ghost + } + if (slice_ghost(ii) or slice_ghost(jj)) { + // if (cid_i == cid_j) { + if (id_i < id_j and slice_ghost(ii)) { + continue; + } else if (id_i > id_j and slice_ghost(jj)) { + continue; + } + //} + } + if (1) { + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) + continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << slice_ghost(ii) << " " + << slice_ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << slice_position(ii, 0) << ", " + << slice_position(ii, 1) << ", " + << slice_position(ii, 2) << " " + << slice_position(jj, 0) << ", " + << slice_position(jj, 1) << ", " + << slice_position(jj, 2) << "\n";// + //std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + //first_neighbor_kernel(ii, jj); + } + } + } // j-loop + }; + + int offset_i = bin_offset(cid_i); + int size_i = bin_size(cid_i); + + for (int i = offset_i; i < offset_i + size_i; ++i) { + // int ii = i; + int ii = cell_list.permutation(i); + int id_i = slice_id(ii); + // if (slice_ghost(ii)) + // continue; + auto p1 = cell_structure.get_local_particle(id_i); + if (p1 == nullptr) + continue; + + if (cid_i == cid_j) { + verlet_kernel(p1, ii, id_i, i + 1, size_i + offset_i - i - 1); // j-loop + //verlet_kernel(p1, i, id_i, i + 1, + // size_i + offset_i - i - 1); // j-loop + } else { + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); + verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop + //verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop + } + } // i-loop + + // Lees-Edwards BC + /*if (le_crossing != 0 && index[le_direction] == 1) { + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + + cell_num[le_direction]) % cell_num[le_direction]; } else if + (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } + cell_offset = bin_offset(dx[0], dx[1], dx[2]); + cell_size = bin_size(dx[0], dx[1], dx[2]); + + verlet_kernel(cell_offset, cell_size); + }*/ + }; + + Kokkos::RangePolicy policy(0, total_pair_cell - + empty_pair_number); + Kokkos::parallel_for("calc_by_cell_list", policy, kernel); + Kokkos::fence(); + } //else { +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List2"); +#endif + +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Calc Forces"); +#endif + Kokkos::RangePolicy policy(0, particle_storage.size()); + Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, + Cabana::FirstNeighborsTag(), + Cabana::SerialOpTag()); + Kokkos::fence(); + //} + + // Save data for next iteration if we just rebuilt + if (rebuild) { + CabanaData new_data(verlet_list, particle_storage.size()); + cell_structure.set_cabana_data(std::make_unique(new_data)); + } // Force and Torque reduction + //Kokkos::RangePolicy policy(0, particle_storage.size()); Kokkos::parallel_for( "reduction", policy, KOKKOS_LAMBDA(const int i) { if (i >= number_of_unique_particles) { @@ -791,7 +796,7 @@ void cabana_short_range( npt_add_virial_force_contribution(virial_vec); #endif #ifdef CALIPER - CALI_MARK_END("Cabana - Execute Kernel"); + CALI_MARK_END("Cabana - Calc Forces"); #endif #ifdef CALIPER @@ -837,41 +842,6 @@ void cabana_short_range( ParticleForce f(f_vec, torque_vec); p->force_and_torque() += f; } - /* - std::unordered_set processed_ids; - - for (auto &p : ghost_particles) { - int const pid = p.id(); - // Check if the particle has already been processed - if (processed_ids.find(pid) != processed_ids.end()) { - continue; - } - - // Check if the ghost particle is in the map, i.e. was used during force - // calculation - if (id_to_index.find(pid) == id_to_index.end()) { - continue; - } - - auto const id = id_to_index.at(pid); - - // Only add forces to ghost particles that are not as normal particles in - // the map, as they have already been added to the force calculation - if (id < particles.size()) { - continue; - } - - processed_ids.insert(pid); - - Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), - slice_force(id, 2)}; - Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), - slice_torque(id, 2)}; - - ParticleForce f(f_vec, torque_vec); - p.force_and_torque() += f; - } - */ #ifdef CALIPER CALI_MARK_END("Cabana - Particle Forces"); #endif From a5e5741b05143fb8093a970ebe237391209facd0 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 16 Jun 2025 13:07:35 +0200 Subject: [PATCH 24/94] Small refactoring --- src/core/communication.cpp | 1 + src/core/short_range_cabana.hpp | 176 ++++++++++++++------------------ 2 files changed, 77 insertions(+), 100 deletions(-) diff --git a/src/core/communication.cpp b/src/core/communication.cpp index cc6bc693d4b..9bc3e0683d5 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -99,6 +99,7 @@ void init(std::shared_ptr mpi_env) { #ifdef SHARED_MEMORY_PARALLELISM Kokkos::initialize(); + //Kokkos::print_configuration(std::cout); #endif } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 726ae2e1e1c..777ee69da42 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -90,6 +90,7 @@ inline void set_offset_and_size_indexed_by_cid( int &total_bins, int* cell_num, cell_list(cid); } } + using ActiveProtocol = std::variant; inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int* cell_num, int* delta_lebc, int le_direction, int le_normal, @@ -105,11 +106,7 @@ inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int* {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, {1, 1, 0}, {1, 1, 1}}; - // std::cout << "TotalBins=" << total_bins << "\n"; - // std::cout << "TotalPairCell=" << total_pair_cell << "\n"; - // - // Creating list of interacting pair cell - // + int empty_pair_number = 0; int pair_cell_id = 0; for (int cid_i = 0; cid_i < total_bins; ++cid_i) { @@ -144,16 +141,25 @@ inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int* cell_num[le_direction]; } } + // Additional Cell + /* + if (le_crossing != 0 && index[le_direction] == 1) { + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } + cell_offset = bin_offset(dx[0], dx[1], dx[2]); + cell_size = bin_size(dx[0], dx[1], dx[2]); + } + */ // Interacting pair cell is registered in the list int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); if (cid_i <= cid_j) { if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { - /*if (pair_cell_id >= total_pair_cell) { - std::cerr << "ERROR: pair_cell_id exceeds total_pair_cell at " - << pair_cell_id << "\n"; - std::terminate(); // or handle safely - }*/ interacting_pair_cell(pair_cell_id, 0) = cid_i; interacting_pair_cell(pair_cell_id, 1) = cid_j; ++pair_cell_id; @@ -214,7 +220,7 @@ void cabana_short_range( // Dont know where to do this better using data_types = Cabana::MemberTypes; - using memory_space = Kokkos::SharedSpace; + using memory_space = Kokkos::HostSpace; //Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using ListAlgorithm = Cabana::HalfNeighborTag; @@ -224,7 +230,7 @@ void cabana_short_range( // Number of threads const int num_threads = execution_space().concurrency(); - const int vector_length = 8; + const int vector_length = 1; #ifdef CALIPER CALI_MARK_END("Cabana - Setup"); #endif @@ -255,17 +261,21 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { + //if (cell_structure.get_local_particle(p.id())) { id_to_index[p.id()] = index; registered_index.insert(p.id()); index++; + //} } for (auto const &p : ghost_particles) { if (not registered_index.contains(p.id())) { + //if (cell_structure.get_local_particle(p.id())) { id_to_index[p.id()] = index; registered_index.insert(p.id()); index++; - } + //} + } } } else { // If we do not rebuild we can use the saved map @@ -297,6 +307,7 @@ void cabana_short_range( int p_id = 0; registered_index.clear(); for (auto const &p : particles) { + //if (!cell_structure.get_local_particle(p.id())) continue; write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); registered_index.insert(p.id()); @@ -308,6 +319,7 @@ void cabana_short_range( if (registered_index.contains(p.id())) { continue; } + //if (!cell_structure.get_local_particle(p.id())) continue; write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); registered_index.insert(p.id()); @@ -423,8 +435,8 @@ void cabana_short_range( auto p1 = cell->get_local_particle(slice_id(i)); auto p2 = cell->get_local_particle(slice_id(j)); - if (p1 == nullptr or p2 == nullptr) - return; + //if (p1 == nullptr or p2 == nullptr) + // return; auto const dist2 = dist * dist; auto[pf, virial] = add_non_bonded_pair_force( @@ -517,10 +529,10 @@ void cabana_short_range( max_counts = static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); } - if (max_counts < 128) - max_counts = 128; + if (max_counts < 256) + max_counts = 256; if (rebuild) { // Legacy Velert List - verlet_list = + /*verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), @@ -536,7 +548,7 @@ void cabana_short_range( //<< p2.pos() << "\n"; }; - cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);// + cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);*/ } else { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); @@ -559,7 +571,7 @@ void cabana_short_range( coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, num_threads, rank, number_of_unique_particles); - if (rebuild and max_cutoff != INACTIVE_CUTOFF and 0) { // Shared memory + if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory // Creating LinkedCellList and VerletList: // Box Properties Cabana::LinkedCellList cell_list; @@ -597,7 +609,7 @@ void cabana_short_range( slice_position, grid_delta, grid_min, grid_max); int total_bins = cell_list.totalBins(); // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. - // Cabana::permute( cell_list, particle_storage ); + //Cabana::permute( cell_list, particle_storage ); verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); @@ -623,7 +635,6 @@ void cabana_short_range( set_interacting_pair_cell(total_bins, total_pair_cell, cell_num, delta_lebc, le_direction, le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); - // End Creating Interacting cell auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; @@ -636,52 +647,45 @@ void cabana_short_range( auto verlet_kernel = [&](Particle *p1, int ii, int id_i, int cell_offset, int cell_size) { for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - // int jj = j; //int ii = cell_list.permutation(i); // debug + //int jj = j; int jj = cell_list.permutation(j); int id_j = slice_id(jj); - if (slice_ghost(ii) and slice_ghost(jj)) { - continue; // reject both ghost - } - if (slice_ghost(ii) or slice_ghost(jj)) { - // if (cid_i == cid_j) { - if (id_i < id_j and slice_ghost(ii)) { - continue; - } else if (id_i > id_j and slice_ghost(jj)) { + if (slice_ghost(ii) or slice_ghost(jj)) { + if ( (id_i < id_j and slice_ghost(ii)) + or (id_i > id_j and slice_ghost(jj)) ) { continue; } - //} + } else if (slice_ghost(ii) and slice_ghost(jj)) { + continue; // reject both ghost } - if (1) { - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) - continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " - << slice_ghost(ii) << " " - << slice_ghost(jj) << " " - << cid_i << " " - << cid_j << " " - << slice_position(ii, 0) << ", " - << slice_position(ii, 1) << ", " - << slice_position(ii, 2) << " " - << slice_position(jj, 0) << ", " - << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";// - //std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ - //first_neighbor_kernel(ii, jj); - } + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << slice_ghost(ii) << " " + << slice_ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << slice_position(ii, 0) << ", " + << slice_position(ii, 1) << ", " + << slice_position(ii, 2) << " " + << slice_position(jj, 0) << ", " + << slice_position(jj, 1) << ", " + << slice_position(jj, 2) << "\n";// + //std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + //first_neighbor_kernel(ii, jj); } } // j-loop }; @@ -690,14 +694,11 @@ void cabana_short_range( int size_i = bin_size(cid_i); for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; + //int ii = i; int ii = cell_list.permutation(i); int id_i = slice_id(ii); - // if (slice_ghost(ii)) - // continue; auto p1 = cell_structure.get_local_particle(id_i); - if (p1 == nullptr) - continue; + if (p1 == nullptr) continue; if (cid_i == cid_j) { verlet_kernel(p1, ii, id_i, i + 1, size_i + offset_i - i - 1); // j-loop @@ -711,19 +712,6 @@ void cabana_short_range( } } // i-loop - // Lees-Edwards BC - /*if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + - cell_num[le_direction]) % cell_num[le_direction]; } else if - (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } - cell_offset = bin_offset(dx[0], dx[1], dx[2]); - cell_size = bin_size(dx[0], dx[1], dx[2]); - - verlet_kernel(cell_offset, cell_size); - }*/ }; Kokkos::RangePolicy policy(0, total_pair_cell - @@ -731,6 +719,13 @@ void cabana_short_range( Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); } //else { + { + Kokkos::RangePolicy policy(0, particle_storage.size()); + Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, + Cabana::FirstNeighborsTag(), + Cabana::SerialOpTag()); + Kokkos::fence(); + } #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List2"); #endif @@ -738,12 +733,6 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Calc Forces"); #endif - Kokkos::RangePolicy policy(0, particle_storage.size()); - Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, - Cabana::FirstNeighborsTag(), - Cabana::SerialOpTag()); - Kokkos::fence(); - //} // Save data for next iteration if we just rebuilt if (rebuild) { @@ -752,14 +741,9 @@ void cabana_short_range( } // Force and Torque reduction - //Kokkos::RangePolicy policy(0, particle_storage.size()); + Kokkos::RangePolicy policy(0, particle_storage.size()); Kokkos::parallel_for( "reduction", policy, KOKKOS_LAMBDA(const int i) { - if (i >= number_of_unique_particles) { - std::cerr << "ERROR: index exceeds number_of_unique_particles " - << i << " " << "\n"; - std::terminate(); // or handle safely - } double fx = 0.; double fy = 0.; double fz = 0.; @@ -809,9 +793,6 @@ void cabana_short_range( collision_detection->detect_collision(p1, p2, d.dist2); } }; - //bool const rebuild_e = cell_structure.get_rebuild_verlet_list(); - //bool const rebuild_c = cell_structure.get_rebuild_cabana_verlet_list(); - //std::cout << "Both should be 0 before non_bonded_loop " << rebuild_e << " " << rebuild_c << std::endl; cell_structure.non_bonded_loop(collision_kernel, verlet_criterion); #endif #ifdef CALIPER @@ -829,11 +810,6 @@ void cabana_short_range( if (p == nullptr) { return; } - if (id >= number_of_unique_particles) { - std::cerr << "ERROR: id exceeds number_of_unique_particles " - << id << " " << "\n"; - std::terminate(); // or handle safely - } Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), slice_force(id, 2)}; Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), From 3104bd78279007884b0e241522d01dadd82eb540 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 16 Jun 2025 13:09:51 +0200 Subject: [PATCH 25/94] Formatting --- src/core/cabana_data.hpp | 6 +- src/core/cell_system/CellStructure.cpp | 10 +- src/core/cell_system/CellStructure.hpp | 25 +-- src/core/communication.cpp | 2 +- src/core/short_range_cabana.hpp | 242 +++++++++++++------------ src/core/system/System.cpp | 2 +- 6 files changed, 148 insertions(+), 139 deletions(-) diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 34b0f5df2fa..23fd7e9ce1e 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -41,8 +41,10 @@ class CabanaData { public: CabanaData() = default; - CabanaData(ListType verlet_list, std::unordered_map id_to_index, std::vector index_to_id) - : verlet_list(verlet_list), id_to_index(id_to_index), index_to_id(index_to_id) {} + CabanaData(ListType verlet_list, std::unordered_map id_to_index, + std::vector index_to_id) + : verlet_list(verlet_list), id_to_index(id_to_index), + index_to_id(index_to_id) {} CabanaData(ListType verlet_list, std::unordered_map id_to_index) : verlet_list(verlet_list), id_to_index(id_to_index) {} CabanaData(ListType verlet_list, int particle_number) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index c81792f4693..2d4b6160838 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -75,8 +75,8 @@ CellStructure::~CellStructure() { void CellStructure::set_cabana_data(std::unique_ptr data) { m_cabana_data = std::move(data); - //m_rebuild_verlet_list = false; - //std::cout << "c1.rebuild " << m_rebuild_verlet_list << std::endl; + // m_rebuild_verlet_list = false; + // std::cout << "c1.rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = false; } @@ -84,8 +84,8 @@ CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } void CellStructure::reset_cabana_data() { m_rebuild_verlet_list = true; - //std::cout << "c2.rebuild " << m_rebuild_verlet_list << std::endl; - //m_rebuild_cabana_verlet_list = true; + // std::cout << "c2.rebuild " << m_rebuild_verlet_list << std::endl; + // m_rebuild_cabana_verlet_list = true; if (m_cabana_data) { m_cabana_data.reset(); } @@ -272,7 +272,7 @@ void CellStructure::resort_particles(bool global_flag) { auto const &lebc = get_system().box_geo->lees_edwards_bc(); m_rebuild_verlet_list = true; - //std::cout << "resort-rebuild " << m_rebuild_verlet_list << std::endl; + // std::cout << "resort-rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = true; m_le_pos_offset_at_last_resort = lebc.pos_offset; diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 983b5c776ae..2cb4a746d99 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +56,6 @@ #include #include #include -#include // forward declaration to not have to import cabana #ifdef SHARED_MEMORY_PARALLELISM @@ -688,20 +688,20 @@ struct CellStructure : public System::Leaf { template void cabana_verlet_list_loop(Kernel kernel, const VerletCriterion &verlet_criterion) { - //if (m_rebuild_cabana_verlet_list) { + // if (m_rebuild_cabana_verlet_list) { if (m_rebuild_verlet_list) { m_verlet_list.clear(); link_cell([&](Particle &p1, Particle &p2, Distance const &d) { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); - //std::cout << "WITHOUT CS " - // << p1.id() << " " - // << p2.id() << std::endl; + // std::cout << "WITHOUT CS " + // << p1.id() << " " + // << p2.id() << std::endl; } }); m_rebuild_verlet_list = false; - //m_rebuild_cabana_verlet_list = false; + // m_rebuild_cabana_verlet_list = false; } for (auto const &pair : m_verlet_list) { kernel(*pair.first, *pair.second); @@ -722,7 +722,8 @@ struct CellStructure : public System::Leaf { /* In this case the verlet list update is attached to * the pair kernel, and the verlet list is rebuilt as * we go. */ - //std::cout << "In verlet_list_looop " << m_rebuild_verlet_list << " " << m_rebuild_cabana_verlet_list << std::endl; + // std::cout << "In verlet_list_looop " << m_rebuild_verlet_list << " " << + // m_rebuild_cabana_verlet_list << std::endl; if (m_rebuild_verlet_list) { m_verlet_list.clear(); @@ -730,14 +731,14 @@ struct CellStructure : public System::Leaf { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); pair_kernel(p1, p2, d); - //std::cout << "WITHOUT CS " - // << p1.id() << " " - // << p2.id() << std::endl; + // std::cout << "WITHOUT CS " + // << p1.id() << " " + // << p2.id() << std::endl; } }); m_rebuild_verlet_list = false; - //std::cout << "h2.rebuild " << m_rebuild_verlet_list << std::endl; + // std::cout << "h2.rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = true; } else { auto const maybe_box = decomposition().minimum_image_distance(); @@ -784,7 +785,7 @@ struct CellStructure : public System::Leaf { template void non_bonded_loop(PairKernel pair_kernel, const VerletCriterion &verlet_criterion) { - //std::cout << "non_bonded_loop " << use_verlet_list << std::endl; + // std::cout << "non_bonded_loop " << use_verlet_list << std::endl; if (use_verlet_list) { verlet_list_loop(pair_kernel, verlet_criterion); } else { diff --git a/src/core/communication.cpp b/src/core/communication.cpp index 9bc3e0683d5..614671b75c8 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -99,7 +99,7 @@ void init(std::shared_ptr mpi_env) { #ifdef SHARED_MEMORY_PARALLELISM Kokkos::initialize(); - //Kokkos::print_configuration(std::cout); + // Kokkos::print_configuration(std::cout); #endif } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 777ee69da42..067bf94d258 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -36,9 +36,9 @@ #include #include #include +#include #include #include -#include inline double wrap(double x, double L) { auto result = x - std::floor(x / L) * L; @@ -73,15 +73,16 @@ inline void write_particle(Particle const &p, int const &id, assert(s_position(id, 2) >= 0. and s_position(id, 2) < box_l[2]); } -inline void set_offset_and_size_indexed_by_cid( int &total_bins, int* cell_num, +inline void set_offset_and_size_indexed_by_cid( + int &total_bins, int *cell_num, Cabana::LinkedCellList &cell_list, Kokkos::View &bin_offset, - Kokkos::View &bin_size) { + Kokkos::View &bin_size) { for (int cid = 0; cid < total_bins; ++cid) { int dx[3] = {}; dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / - cell_num[2]); + cell_num[2]); dx[2] = cid % cell_num[2]; bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); @@ -91,9 +92,11 @@ inline void set_offset_and_size_indexed_by_cid( int &total_bins, int* cell_num, } } -using ActiveProtocol = std::variant; -inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int* cell_num, - int* delta_lebc, int le_direction, int le_normal, +using ActiveProtocol = std::variant; +inline int set_interacting_pair_cell( + int &total_bins, int total_pair_cell, int *cell_num, int *delta_lebc, + int le_direction, int le_normal, std::shared_ptr le_protocol, Kokkos::View &bin_size, Cabana::LinkedCellList &cell_list, @@ -119,33 +122,33 @@ inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int* bool duplicate_cell = false; // Obtaining 3 dimentional cell index from neighbor cell for (int d = 0; d < 3; ++d) { - dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; - if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) - duplicate_cell = true; + dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; + if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) + duplicate_cell = true; } if (duplicate_cell) - continue; + continue; // Lees-Edwards BC int le_crossing = 0; if (le_protocol != nullptr) { - le_crossing = - ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } + le_crossing = + ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } } // Additional Cell /* if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + cell_num[le_direction]) % cell_num[le_direction]; } else if (le_crossing > 0) { dx[le_direction] = (dx[le_direction] - 1 + @@ -159,20 +162,19 @@ inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int* // Interacting pair cell is registered in the list int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); if (cid_i <= cid_j) { - if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - ++pair_cell_id; - } else { - ++empty_pair_number; - } + if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } else { + ++empty_pair_number; + } } } } return empty_pair_number; } - template void cabana_short_range( BondKernel bond_kernel, @@ -220,7 +222,7 @@ void cabana_short_range( // Dont know where to do this better using data_types = Cabana::MemberTypes; - using memory_space = Kokkos::HostSpace; //Kokkos::SharedSpace; + using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using ListAlgorithm = Cabana::HalfNeighborTag; @@ -247,7 +249,7 @@ void cabana_short_range( int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); - //std::cout << "For CABANA rebuild " << rebuild << std::endl; + // std::cout << "For CABANA rebuild " << rebuild << std::endl; CabanaData saved_data; @@ -261,7 +263,7 @@ void cabana_short_range( if (rebuild) { for (auto const &p : particles) { - //if (cell_structure.get_local_particle(p.id())) { + // if (cell_structure.get_local_particle(p.id())) { id_to_index[p.id()] = index; registered_index.insert(p.id()); index++; @@ -270,16 +272,16 @@ void cabana_short_range( for (auto const &p : ghost_particles) { if (not registered_index.contains(p.id())) { - //if (cell_structure.get_local_particle(p.id())) { + // if (cell_structure.get_local_particle(p.id())) { id_to_index[p.id()] = index; registered_index.insert(p.id()); index++; //} - } + } } } else { // If we do not rebuild we can use the saved map - //id_to_index = saved_data.get_id_to_index(); + // id_to_index = saved_data.get_id_to_index(); index = saved_data.get_index(); } @@ -307,7 +309,7 @@ void cabana_short_range( int p_id = 0; registered_index.clear(); for (auto const &p : particles) { - //if (!cell_structure.get_local_particle(p.id())) continue; + // if (!cell_structure.get_local_particle(p.id())) continue; write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); registered_index.insert(p.id()); @@ -319,7 +321,7 @@ void cabana_short_range( if (registered_index.contains(p.id())) { continue; } - //if (!cell_structure.get_local_particle(p.id())) continue; + // if (!cell_structure.get_local_particle(p.id())) continue; write_particle(p, p_id, slice_position, slice_force, slice_torque, slice_charge, slice_id, slice_type, slice_ghost, box_l); registered_index.insert(p.id()); @@ -346,7 +348,7 @@ void cabana_short_range( CALI_MARK_END("Cabana - Fill particle storage"); #endif - // The kernel of calculate force + // The kernel of calculate force struct FirstNeighborKernel { const CellStructure *cell; [[maybe_unused]] const BondedInteractionsMap &bonded_ias; @@ -409,7 +411,8 @@ void cabana_short_range( #endif coulomb_kernel(coulomb_kernel_), dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), - num_threads(num_threads_), mpi_rank(mpi_rank_), particle_number(particle_number_) { + num_threads(num_threads_), mpi_rank(mpi_rank_), + particle_number(particle_number_) { } KOKKOS_INLINE_FUNCTION @@ -427,7 +430,8 @@ void cabana_short_range( auto thread_id = omp_get_thread_num(); // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - //std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << " " << + // std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << + // " " << IA_parameters const &ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); @@ -435,14 +439,14 @@ void cabana_short_range( auto p1 = cell->get_local_particle(slice_id(i)); auto p2 = cell->get_local_particle(slice_id(j)); - //if (p1 == nullptr or p2 == nullptr) - // return; + // if (p1 == nullptr or p2 == nullptr) + // return; auto const dist2 = dist * dist; - auto[pf, virial] = add_non_bonded_pair_force( - const_cast(*p1), const_cast(*p2), - d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, - coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); + auto [pf, virial] = add_non_bonded_pair_force( + const_cast(*p1), const_cast(*p2), d, dist, + dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, + coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); // /* ParticleForce pf{}; @@ -522,7 +526,7 @@ void cabana_short_range( // Rebuild verlet list if needed auto const &system = ::System::get_system(); int max_counts; - double max_cutoff = pair_cutoff; //system.get_interaction_range(); + double max_cutoff = pair_cutoff; // system.get_interaction_range(); if (std::isinf(max_cutoff)) { max_counts = number_of_unique_particles; } else { @@ -581,11 +585,11 @@ void cabana_short_range( int cell_num[3] = {}; double eff_cutoff; for (int d = 0; d < 3; ++d) { - eff_cutoff = max_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + eff_cutoff = max_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); } // For Lees-Edwards boundary condition double le_offset; @@ -594,22 +598,23 @@ void cabana_short_range( int delta_lebc[3] = {0, 0, 0}; auto le_protocol = system.lees_edwards->get_protocol(); if (le_protocol == nullptr) { - le_offset = 0.; - le_direction = -1; - le_normal = -1; + le_offset = 0.; + le_direction = -1; + le_normal = -1; } else { - le_offset = box_geo.lees_edwards_bc().pos_offset; - le_direction = box_geo.lees_edwards_bc().shear_direction; - le_normal = box_geo.lees_edwards_bc().shear_plane_normal; - delta_lebc[le_direction] = - static_cast(std::ceil(le_offset / grid_delta[le_direction])) % - cell_num[le_direction]; + le_offset = box_geo.lees_edwards_bc().pos_offset; + le_direction = box_geo.lees_edwards_bc().shear_direction; + le_normal = box_geo.lees_edwards_bc().shear_plane_normal; + delta_lebc[le_direction] = + static_cast(std::ceil(le_offset / grid_delta[le_direction])) % + cell_num[le_direction]; } cell_list = Cabana::createLinkedCellList( - slice_position, grid_delta, grid_min, grid_max); + slice_position, grid_delta, grid_min, grid_max); int total_bins = cell_list.totalBins(); - // Now permute the AoSoA (i.e. reorder the data) using the linked cell list. - //Cabana::permute( cell_list, particle_storage ); + // Now permute the AoSoA (i.e. reorder the data) using the linked cell + // list. + // Cabana::permute( cell_list, particle_storage ); verlet_list = ListType(slice_position, 0, slice_position.size(), max_counts); @@ -618,10 +623,10 @@ void cabana_short_range( Kokkos::View bin_offset("bin_offset", total_bins); Kokkos::View bin_size("bin_size", total_bins); - set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, bin_offset, bin_size); + set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, + bin_offset, bin_size); auto const particle_bins = cell_list.getParticleBins(); - // Creating Interacting cell int total_pair_cell; if (total_bins < 27) { @@ -631,10 +636,9 @@ void cabana_short_range( } Kokkos::View interacting_pair_cell( "interacting_pair_cell", total_pair_cell, 2); - int empty_pair_number = - set_interacting_pair_cell(total_bins, total_pair_cell, cell_num, - delta_lebc, le_direction, le_normal, le_protocol, - bin_size, cell_list, interacting_pair_cell); + int empty_pair_number = set_interacting_pair_cell( + total_bins, total_pair_cell, cell_num, delta_lebc, le_direction, + le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; @@ -644,48 +648,49 @@ void cabana_short_range( int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); - auto verlet_kernel = [&](Particle *p1, int ii, int id_i, int cell_offset, - int cell_size) { + auto verlet_kernel = [&](Particle *p1, int ii, int id_i, + int cell_offset, int cell_size) { for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - //int ii = cell_list.permutation(i); // debug - //int jj = j; + // int ii = cell_list.permutation(i); // debug + // int jj = j; int jj = cell_list.permutation(j); int id_j = slice_id(jj); - if (slice_ghost(ii) or slice_ghost(jj)) { - if ( (id_i < id_j and slice_ghost(ii)) - or (id_i > id_j and slice_ghost(jj)) ) { + if (slice_ghost(ii) or slice_ghost(jj)) { + if ((id_i < id_j and slice_ghost(ii)) or + (id_i > id_j and slice_ghost(jj))) { continue; } - } else if (slice_ghost(ii) and slice_ghost(jj)) { + } else if (slice_ghost(ii) and slice_ghost(jj)) { continue; // reject both ghost } - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " - << slice_ghost(ii) << " " - << slice_ghost(jj) << " " - << cid_i << " " - << cid_j << " " - << slice_position(ii, 0) << ", " - << slice_position(ii, 1) << ", " - << slice_position(ii, 2) << " " - << slice_position(jj, 0) << ", " - << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";// - //std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ - //first_neighbor_kernel(ii, jj); + auto p2 = cell_structure.get_local_particle(id_j); + if (p2 == nullptr) + continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << slice_ghost(ii) << " " + << slice_ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << slice_position(ii, 0) << ", " + << slice_position(ii, 1) << ", " + << slice_position(ii, 2) << " " + << slice_position(jj, 0) << ", " + << slice_position(jj, 1) << ", " + << slice_position(jj, 2) << "\n";// + //std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + // first_neighbor_kernel(ii, jj); } } // j-loop }; @@ -694,36 +699,37 @@ void cabana_short_range( int size_i = bin_size(cid_i); for (int i = offset_i; i < offset_i + size_i; ++i) { - //int ii = i; + // int ii = i; int ii = cell_list.permutation(i); int id_i = slice_id(ii); auto p1 = cell_structure.get_local_particle(id_i); - if (p1 == nullptr) continue; + if (p1 == nullptr) + continue; if (cid_i == cid_j) { - verlet_kernel(p1, ii, id_i, i + 1, size_i + offset_i - i - 1); // j-loop - //verlet_kernel(p1, i, id_i, i + 1, - // size_i + offset_i - i - 1); // j-loop + verlet_kernel(p1, ii, id_i, i + 1, + size_i + offset_i - i - 1); // j-loop + // verlet_kernel(p1, i, id_i, i + 1, + // size_i + offset_i - i - 1); // j-loop } else { int offset_j = bin_offset(cid_j); int size_j = bin_size(cid_j); verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop - //verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop + // verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop } } // i-loop - }; Kokkos::RangePolicy policy(0, total_pair_cell - empty_pair_number); Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); - } //else { + } // else { { Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, - Cabana::FirstNeighborsTag(), - Cabana::SerialOpTag()); + Cabana::FirstNeighborsTag(), + Cabana::SerialOpTag()); Kokkos::fence(); } #ifdef CALIPER diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index fce28161761..c2daa314b2a 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -95,7 +95,7 @@ System::System(Private) { } System::~System() { - //std::cout << "~System()\n"; + // std::cout << "~System()\n"; #ifdef SHARED_MEMORY_PARALLELISM cell_structure->reset_cabana_data(); #endif From 31ca6a97b3336ec86355d625b4074f17044e3a88 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 16 Jun 2025 16:06:26 +0200 Subject: [PATCH 26/94] Ingnored modernize-use-nullptr in Cabana_Parallel.hpp --- src/core/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 9fe2c4e6057..cb8e797a615 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -108,10 +108,12 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) endif() if(ESPRESSO_BUILD_WITH_WALBERLA) - target_link_libraries( - espresso_core - PRIVATE espresso::walberla - $<$:espresso::walberla_cuda>) + set_source_files_properties( + ${CMAKE_CURRENT_SOURCE_DIR}/forces.cpp + PROPERTIES + CMAKE_CXX_CLANG_TIDY + "/usr/bin/clang-tidy-19;-checks=*,-modernize-use-nullptr,-clang-analyzer-optin.performance.Padding" + ) endif() if(ESPRESSO_BUILD_WITH_FFTW) From 75bada73e19d55f60342e1f351839b4bf51337b9 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 16 Jun 2025 16:08:53 +0200 Subject: [PATCH 27/94] Formatting --- src/core/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index cb8e797a615..ec1be1c4ec7 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -109,8 +109,8 @@ endif() if(ESPRESSO_BUILD_WITH_WALBERLA) set_source_files_properties( - ${CMAKE_CURRENT_SOURCE_DIR}/forces.cpp - PROPERTIES + ${CMAKE_CURRENT_SOURCE_DIR}/forces.cpp + PROPERTIES CMAKE_CXX_CLANG_TIDY "/usr/bin/clang-tidy-19;-checks=*,-modernize-use-nullptr,-clang-analyzer-optin.performance.Padding" ) From f40d4f5487366145281f2ba771d4022241a2f1b9 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 16 Jun 2025 16:33:01 +0200 Subject: [PATCH 28/94] Corrected CMakeLists.txt --- src/core/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index ec1be1c4ec7..1618680f48a 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -108,6 +108,10 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) endif() if(ESPRESSO_BUILD_WITH_WALBERLA) + target_link_libraries( + espresso_core + PRIVATE espresso::walberla + $<$:espresso::walberla_cuda>) set_source_files_properties( ${CMAKE_CURRENT_SOURCE_DIR}/forces.cpp PROPERTIES From 970bbaaed6dec101630e732fe7c1a530703bb6ec Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 16 Jun 2025 16:33:57 +0200 Subject: [PATCH 29/94] Formatting --- src/core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1618680f48a..755d0263219 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -111,7 +111,7 @@ if(ESPRESSO_BUILD_WITH_WALBERLA) target_link_libraries( espresso_core PRIVATE espresso::walberla - $<$:espresso::walberla_cuda>) + $<$:espresso::walberla_cuda>) set_source_files_properties( ${CMAKE_CURRENT_SOURCE_DIR}/forces.cpp PROPERTIES From e6afa567be483201ffbcbe33e4713b6676c9aba1 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 17 Jun 2025 19:57:19 +0200 Subject: [PATCH 30/94] Modified ifdef in forces_inline --- CMakeLists.txt | 4 ++++ src/core/communication.cpp | 2 +- src/core/forces_inline.hpp | 24 +++++++++++++------ src/core/short_range_cabana.hpp | 41 ++++++++++++++++++--------------- 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bc73c4f9cb..6f230a076b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -547,6 +547,9 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "") set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "") set(Kokkos_ENABLE_IMPL_VIEW_LEGACY ON CACHE BOOL "") + set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "") + set(Kokkos_ENABLE_HWLOC ON CACHE BOOL "") + set(Kokkos_ARCH_ZEN4 ON CACHE BOOL "") FetchContent_MakeAvailable(kokkos) set(BUILD_SHARED_LIBS ${ESPRESSO_BUILD_SHARED_LIBS_DEFAULT}) set(CMAKE_SHARED_LIBRARY_PREFIX "${ESPRESSO_SHARED_LIBRARY_PREFIX}") @@ -554,6 +557,7 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) # install all kokkos shared objects get_target_property(ESPRESSO_KOKKOS_LIBS kokkos INTERFACE_LINK_LIBRARIES) foreach(target_name IN LISTS ESPRESSO_KOKKOS_LIBS) + message(${target_name}) get_target_property(target_type ${target_name} TYPE) if(${target_type} STREQUAL "SHARED_LIBRARY" AND ${target_name} MATCHES "^kokkos[a-zA-Z0-9_]+$") diff --git a/src/core/communication.cpp b/src/core/communication.cpp index 614671b75c8..9586fe1c0df 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -99,7 +99,7 @@ void init(std::shared_ptr mpi_env) { #ifdef SHARED_MEMORY_PARALLELISM Kokkos::initialize(); - // Kokkos::print_configuration(std::cout); + Kokkos::print_configuration(std::cout); #endif } diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 1344db61cf6..71f6f2e932e 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -213,7 +213,10 @@ inline void add_non_bonded_pair_withot_p( * For the interaction which need particle information */ inline void add_non_bonded_pair_force_with_p( - Particle &p1, Particle &p2, ParticleForce &pf, Utils::Vector3d &virial, + Particle &p1, Particle &p2, ParticleForce &pf, +#ifdef SHARED_MEMORY_PARALLELISM + Utils::Vector3d &virial, +#endif Utils::Vector3d const &d, double dist, double dist2, double q1q2, IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, @@ -308,8 +311,10 @@ inline void add_non_bonded_pair_force_with_p( // return std::pair{pf, virial}; } -#ifdef SHARED_MEMORY_PARALLELISM +#if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) using ReturnType = std::pair; +#elif defined(SHARED_MEMORY_PARALLELISM) +using ReturnType = ParticleForce; #else using ReturnType = void; #endif @@ -341,8 +346,7 @@ inline ReturnType add_non_bonded_pair_force( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { ParticleForce pf{}; -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(SHARED_MEMORY_PARALLELISM) +#if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) Utils::Vector3d virial{}; #endif @@ -355,10 +359,14 @@ inline ReturnType add_non_bonded_pair_force( add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, coulomb_kernel); -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ +#if defined(NPT) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) add_non_bonded_pair_force_with_p( - p1, p2, pf, virial, d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, + p1, p2, pf, +#if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) + virial, +#endif + d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); #endif @@ -367,8 +375,10 @@ inline ReturnType add_non_bonded_pair_force( /* add total non-bonded forces to particles */ /***********************************************/ -#ifdef SHARED_MEMORY_PARALLELISM +#if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) return std::pair{pf, virial}; +#elif defined(SHARED_MEMORY_PARALLELISM) + return pf; #else p1.force_and_torque() += pf; p2.force_and_torque() += calc_opposing_force(pf, d); diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 067bf94d258..aa61d7ccbe6 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -249,7 +249,7 @@ void cabana_short_range( int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); - // std::cout << "For CABANA rebuild " << rebuild << std::endl; + //std::cout << "For CABANA rebuild " << rebuild << " " << Kokkos::OpenMP::concurrency() << std::endl; CabanaData saved_data; @@ -435,7 +435,7 @@ void cabana_short_range( IA_parameters const &ia_params = nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); - // + /* auto p1 = cell->get_local_particle(slice_id(i)); auto p2 = cell->get_local_particle(slice_id(j)); @@ -447,10 +447,12 @@ void cabana_short_range( const_cast(*p1), const_cast(*p2), d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); - // - /* + */ + ParticleForce pf{}; +#ifdef NPT Utils::Vector3d virial{}; +#endif #ifdef EXCLUSIONS auto p1 = cell->get_local_particle(slice_id(i)); @@ -468,7 +470,7 @@ void cabana_short_range( do_nonbonded_flag, coulomb_kernel); #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) + defined(DPD) or defined(DIPOLES) or defined(NPT) auto const dist2 = dist * dist; #ifndef EXCLUSIONS @@ -477,14 +479,17 @@ void cabana_short_range( if (p1 == nullptr or p2 == nullptr) return; -#endif +#endif // NOT EXCLUSIONS add_non_bonded_pair_force_with_p( const_cast(*p1), const_cast(*p2), pf, - virial, d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, +#ifdef NPT + virial, +#endif //NPT + d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); #endif // ETC - */ + // local_force(thread_id, i, 0) += pf.f[0]; local_force(thread_id, i, 1) += pf.f[1]; local_force(thread_id, i, 2) += pf.f[2]; @@ -690,7 +695,7 @@ void cabana_short_range( << dx[0] << " " << dx[1] << " " << dx[2] << "\n";*/ - // first_neighbor_kernel(ii, jj); + first_neighbor_kernel(ii, jj); } } // j-loop }; @@ -724,28 +729,28 @@ void cabana_short_range( empty_pair_number); Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); - } // else { - { + } else { + //{ Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), Cabana::SerialOpTag()); + //Cabana::TeamOpTag()); Kokkos::fence(); } -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List2"); -#endif - -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Calc Forces"); -#endif // Save data for next iteration if we just rebuilt if (rebuild) { CabanaData new_data(verlet_list, particle_storage.size()); cell_structure.set_cabana_data(std::make_unique(new_data)); } +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List2"); +#endif +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Calc Forces"); +#endif // Force and Torque reduction Kokkos::RangePolicy policy(0, particle_storage.size()); Kokkos::parallel_for( From 01160e068da67ebeeda66c35dce43a88e8216dd4 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 17 Jun 2025 22:30:55 +0200 Subject: [PATCH 31/94] Fixed compiling Kokkos with clang_tidy --- CMakeLists.txt | 13 ++++++++++++- src/core/CMakeLists.txt | 11 ----------- src/core/cell_system/CellStructure.cpp | 5 ----- src/core/cell_system/CellStructure.hpp | 11 ----------- src/core/communication.cpp | 2 +- src/core/short_range_cabana.hpp | 15 +++++++-------- src/core/system/System.cpp | 1 - 7 files changed, 20 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f230a076b7..a045844dd03 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -557,7 +557,6 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) # install all kokkos shared objects get_target_property(ESPRESSO_KOKKOS_LIBS kokkos INTERFACE_LINK_LIBRARIES) foreach(target_name IN LISTS ESPRESSO_KOKKOS_LIBS) - message(${target_name}) get_target_property(target_type ${target_name} TYPE) if(${target_type} STREQUAL "SHARED_LIBRARY" AND ${target_name} MATCHES "^kokkos[a-zA-Z0-9_]+$") @@ -586,6 +585,8 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) if(ESPRESSO_BUILD_WITH_CLANG_TIDY) # silence Kokkos and Cabana diagnostics + set(KOKKOS_CXX_CLANG_TIDY "${ESPRESSO_CXX_CLANG_TIDY}") + set(KOKKOS_CUDA_CLANG_TIDY "${ESPRESSO_CUDA_CLANG_TIDY}") set(CABANA_CXX_CLANG_TIDY "${ESPRESSO_CXX_CLANG_TIDY}") set(CABANA_CUDA_CLANG_TIDY "${ESPRESSO_CUDA_CLANG_TIDY}") unset(SKIP_CLANG_TIDY_CHECKS) @@ -608,6 +609,16 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) list(APPEND SKIP_CLANG_TIDY_CHECKS "-readability-simplify-boolean-expr") list(APPEND SKIP_CLANG_TIDY_CHECKS "-readability-avoid-const-params-in-decls") + list(APPEND SKIP_CLANG_TIDY_CHECKS + "-clang-analyzer-optin.performance.Padding") + list(APPEND SKIP_CLANG_TIDY_CHECKS + "-clang-analyzer-security.insecureAPI.strcpy") + espresso_override_clang_tidy_checks( + KOKKOS_CXX_CLANG_TIDY "${SKIP_CLANG_TIDY_CHECKS}" + "${SKIP_CLANG_TIDY_CHECKS_CXX}") + espresso_override_clang_tidy_checks( + KOKKOS_CUDA_CLANG_TIDY "${SKIP_CLANG_TIDY_CHECKS}" + "${SKIP_CLANG_TIDY_CHECKS_CUDA}") espresso_override_clang_tidy_checks( CABANA_CXX_CLANG_TIDY "${SKIP_CLANG_TIDY_CHECKS}" "${SKIP_CLANG_TIDY_CHECKS_CXX}") diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 755d0263219..da836dcf977 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -101,23 +101,12 @@ target_link_libraries( Boost::serialization Boost::mpi espresso::instrumentation) target_include_directories(espresso_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) - target_include_directories( - espresso_core SYSTEM INTERFACE ${FETCHCONTENT_BASE_DIR}/cabana-src/core/src - ${FETCHCONTENT_BASE_DIR}/kokkos-src/core/src) -endif() if(ESPRESSO_BUILD_WITH_WALBERLA) target_link_libraries( espresso_core PRIVATE espresso::walberla $<$:espresso::walberla_cuda>) - set_source_files_properties( - ${CMAKE_CURRENT_SOURCE_DIR}/forces.cpp - PROPERTIES - CMAKE_CXX_CLANG_TIDY - "/usr/bin/clang-tidy-19;-checks=*,-modernize-use-nullptr,-clang-analyzer-optin.performance.Padding" - ) endif() if(ESPRESSO_BUILD_WITH_FFTW) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 2d4b6160838..7bca218b819 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -75,8 +75,6 @@ CellStructure::~CellStructure() { void CellStructure::set_cabana_data(std::unique_ptr data) { m_cabana_data = std::move(data); - // m_rebuild_verlet_list = false; - // std::cout << "c1.rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = false; } @@ -84,8 +82,6 @@ CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } void CellStructure::reset_cabana_data() { m_rebuild_verlet_list = true; - // std::cout << "c2.rebuild " << m_rebuild_verlet_list << std::endl; - // m_rebuild_cabana_verlet_list = true; if (m_cabana_data) { m_cabana_data.reset(); } @@ -272,7 +268,6 @@ void CellStructure::resort_particles(bool global_flag) { auto const &lebc = get_system().box_geo->lees_edwards_bc(); m_rebuild_verlet_list = true; - // std::cout << "resort-rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = true; m_le_pos_offset_at_last_resort = lebc.pos_offset; diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 2cb4a746d99..147e949d702 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -695,13 +695,9 @@ struct CellStructure : public System::Leaf { link_cell([&](Particle &p1, Particle &p2, Distance const &d) { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); - // std::cout << "WITHOUT CS " - // << p1.id() << " " - // << p2.id() << std::endl; } }); m_rebuild_verlet_list = false; - // m_rebuild_cabana_verlet_list = false; } for (auto const &pair : m_verlet_list) { kernel(*pair.first, *pair.second); @@ -722,8 +718,6 @@ struct CellStructure : public System::Leaf { /* In this case the verlet list update is attached to * the pair kernel, and the verlet list is rebuilt as * we go. */ - // std::cout << "In verlet_list_looop " << m_rebuild_verlet_list << " " << - // m_rebuild_cabana_verlet_list << std::endl; if (m_rebuild_verlet_list) { m_verlet_list.clear(); @@ -731,14 +725,10 @@ struct CellStructure : public System::Leaf { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); pair_kernel(p1, p2, d); - // std::cout << "WITHOUT CS " - // << p1.id() << " " - // << p2.id() << std::endl; } }); m_rebuild_verlet_list = false; - // std::cout << "h2.rebuild " << m_rebuild_verlet_list << std::endl; m_rebuild_cabana_verlet_list = true; } else { auto const maybe_box = decomposition().minimum_image_distance(); @@ -785,7 +775,6 @@ struct CellStructure : public System::Leaf { template void non_bonded_loop(PairKernel pair_kernel, const VerletCriterion &verlet_criterion) { - // std::cout << "non_bonded_loop " << use_verlet_list << std::endl; if (use_verlet_list) { verlet_list_loop(pair_kernel, verlet_criterion); } else { diff --git a/src/core/communication.cpp b/src/core/communication.cpp index 9586fe1c0df..9bc3e0683d5 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -99,7 +99,7 @@ void init(std::shared_ptr mpi_env) { #ifdef SHARED_MEMORY_PARALLELISM Kokkos::initialize(); - Kokkos::print_configuration(std::cout); + //Kokkos::print_configuration(std::cout); #endif } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index aa61d7ccbe6..130dd2ea3c9 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -524,7 +524,7 @@ void cabana_short_range( // Get Verlet Pairs and Fill list // =================================================== #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List1"); + CALI_MARK_BEGIN("Cabana - Verlet List by ESPRESSO"); #endif ListType verlet_list; @@ -563,10 +563,10 @@ void cabana_short_range( verlet_list = saved_data.get_verlet_list(); } #ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List1"); + CALI_MARK_END("Cabana - Verlet List by ESPRESSO"); #endif #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List2"); + CALI_MARK_BEGIN("Cabana - Verlet List and calc Force"); #endif FirstNeighborKernel first_neighbor_kernel( @@ -734,8 +734,7 @@ void cabana_short_range( Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), - Cabana::SerialOpTag()); - //Cabana::TeamOpTag()); + Cabana::TeamOpTag()); Kokkos::fence(); } @@ -745,11 +744,11 @@ void cabana_short_range( cell_structure.set_cabana_data(std::make_unique(new_data)); } #ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List2"); + CALI_MARK_END("Cabana - Verlet List and calc Force"); #endif #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Calc Forces"); + CALI_MARK_BEGIN("Cabana - reduction Forces"); #endif // Force and Torque reduction Kokkos::RangePolicy policy(0, particle_storage.size()); @@ -791,7 +790,7 @@ void cabana_short_range( npt_add_virial_force_contribution(virial_vec); #endif #ifdef CALIPER - CALI_MARK_END("Cabana - Calc Forces"); + CALI_MARK_END("Cabana - reduction Forces"); #endif #ifdef CALIPER diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index c2daa314b2a..cb5d7851da1 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -95,7 +95,6 @@ System::System(Private) { } System::~System() { - // std::cout << "~System()\n"; #ifdef SHARED_MEMORY_PARALLELISM cell_structure->reset_cabana_data(); #endif From be95f0a644f2f49fc5f2dd11da1b643bce156752 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 17 Jun 2025 22:33:23 +0200 Subject: [PATCH 32/94] Formatting --- CMakeLists.txt | 4 ++-- src/core/communication.cpp | 2 +- src/core/forces_inline.hpp | 11 +++++------ src/core/short_range_cabana.hpp | 15 ++++++++------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a045844dd03..622928b60c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -610,9 +610,9 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) list(APPEND SKIP_CLANG_TIDY_CHECKS "-readability-avoid-const-params-in-decls") list(APPEND SKIP_CLANG_TIDY_CHECKS - "-clang-analyzer-optin.performance.Padding") + "-clang-analyzer-optin.performance.Padding") list(APPEND SKIP_CLANG_TIDY_CHECKS - "-clang-analyzer-security.insecureAPI.strcpy") + "-clang-analyzer-security.insecureAPI.strcpy") espresso_override_clang_tidy_checks( KOKKOS_CXX_CLANG_TIDY "${SKIP_CLANG_TIDY_CHECKS}" "${SKIP_CLANG_TIDY_CHECKS_CXX}") diff --git a/src/core/communication.cpp b/src/core/communication.cpp index 9bc3e0683d5..614671b75c8 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -99,7 +99,7 @@ void init(std::shared_ptr mpi_env) { #ifdef SHARED_MEMORY_PARALLELISM Kokkos::initialize(); - //Kokkos::print_configuration(std::cout); + // Kokkos::print_configuration(std::cout); #endif } diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 71f6f2e932e..b72582734f1 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -213,7 +213,7 @@ inline void add_non_bonded_pair_withot_p( * For the interaction which need particle information */ inline void add_non_bonded_pair_force_with_p( - Particle &p1, Particle &p2, ParticleForce &pf, + Particle &p1, Particle &p2, ParticleForce &pf, #ifdef SHARED_MEMORY_PARALLELISM Utils::Vector3d &virial, #endif @@ -359,16 +359,15 @@ inline ReturnType add_non_bonded_pair_force( add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, coulomb_kernel); -#if defined(NPT) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) +#if defined(NPT) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) add_non_bonded_pair_force_with_p( p1, p2, pf, #if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) virial, #endif - d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, - thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, - elc_kernel, coulomb_u_kernel); + d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, + bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); #endif /***********************************************/ diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 130dd2ea3c9..bac43596562 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -249,7 +249,8 @@ void cabana_short_range( int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); - //std::cout << "For CABANA rebuild " << rebuild << " " << Kokkos::OpenMP::concurrency() << std::endl; + // std::cout << "For CABANA rebuild " << rebuild << " " << + // Kokkos::OpenMP::concurrency() << std::endl; CabanaData saved_data; @@ -448,7 +449,7 @@ void cabana_short_range( dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); */ - + ParticleForce pf{}; #ifdef NPT Utils::Vector3d virial{}; @@ -484,10 +485,10 @@ void cabana_short_range( const_cast(*p1), const_cast(*p2), pf, #ifdef NPT virial, -#endif //NPT - d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, - thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, - elc_kernel, coulomb_u_kernel); +#endif // NPT + d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, + box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, + coulomb_u_kernel); #endif // ETC // local_force(thread_id, i, 0) += pf.f[0]; @@ -730,7 +731,7 @@ void cabana_short_range( Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); } else { - //{ + //{ Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), From 6f3ecf33b98d7052a4137ce399a2d9b183978c91 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 17 Jun 2025 22:50:23 +0200 Subject: [PATCH 33/94] Fixed warning: modernize-use-nullptr in Kokkos --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 622928b60c7..0e0d40b7fd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -549,7 +549,7 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) set(Kokkos_ENABLE_IMPL_VIEW_LEGACY ON CACHE BOOL "") set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "") set(Kokkos_ENABLE_HWLOC ON CACHE BOOL "") - set(Kokkos_ARCH_ZEN4 ON CACHE BOOL "") + set(Kokkos_ARCH_NATIVE ON CACHE BOOL "") FetchContent_MakeAvailable(kokkos) set(BUILD_SHARED_LIBS ${ESPRESSO_BUILD_SHARED_LIBS_DEFAULT}) set(CMAKE_SHARED_LIBRARY_PREFIX "${ESPRESSO_SHARED_LIBRARY_PREFIX}") @@ -601,6 +601,7 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) list(APPEND SKIP_CLANG_TIDY_CHECKS "-modernize-use-bool-literals") list(APPEND SKIP_CLANG_TIDY_CHECKS "-modernize-use-equals-delete") list(APPEND SKIP_CLANG_TIDY_CHECKS "-modernize-use-equals-default") + list(APPEND SKIP_CLANG_TIDY_CHECKS "-modernize-use-nullptr") list(APPEND SKIP_CLANG_TIDY_CHECKS "-modernize-pass-by-value") list(APPEND SKIP_CLANG_TIDY_CHECKS "-modernize-loop-convert") list(APPEND SKIP_CLANG_TIDY_CHECKS "-modernize-return-braced-init-list") From c5d43d37f0be707c47fb539cea330c5ab10d1541 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 20 Jun 2025 08:21:27 +0200 Subject: [PATCH 34/94] Refactoring --- CMakeLists.txt | 1 + src/core/aosoa_pack.hpp | 53 +++ src/core/cabana_data.hpp | 4 + src/core/short_range_cabana.hpp | 582 ++++++++++---------------------- src/core/verlet_list_loop.hpp | 317 +++++++++++++++++ testsuite/python/exclusions.py | 1 - 6 files changed, 561 insertions(+), 397 deletions(-) create mode 100644 src/core/aosoa_pack.hpp create mode 100644 src/core/verlet_list_loop.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e0d40b7fd4..ba156439142 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -547,6 +547,7 @@ if(ESPRESSO_BUILD_WITH_SHARED_MEMORY_PARALLELISM) set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "") set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "") set(Kokkos_ENABLE_IMPL_VIEW_LEGACY ON CACHE BOOL "") + set(Kokkos_ENABLE_COMPLEX_ALIGN ON CACHE BOOL "") set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "") set(Kokkos_ENABLE_HWLOC ON CACHE BOOL "") set(Kokkos_ARCH_NATIVE ON CACHE BOOL "") diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp new file mode 100644 index 00000000000..9a88ff02049 --- /dev/null +++ b/src/core/aosoa_pack.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2010-2025 The ESPResSo project + * + * This file is part of ESPResSo. + * + * ESPResSo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ESPResSo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#ifdef SHARED_MEMORY_PARALLELISM + +#include + +const int vector_length = 1; +using data_types = Cabana::MemberTypes; +using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; +using execution_space = Kokkos::DefaultExecutionSpace; +using AoSoA_type = Cabana::AoSoA; + +struct AoSoA_pack { + AoSoA_type::member_slice_type<0> position; + AoSoA_type::member_slice_type<1> force; + AoSoA_type::member_slice_type<2> torque; + AoSoA_type::member_slice_type<3> charge; + AoSoA_type::member_slice_type<4> id; + AoSoA_type::member_slice_type<5> type; + AoSoA_type::member_slice_type<6> ghost; + + AoSoA_pack() = default; + + AoSoA_pack(AoSoA_type &aosoa) + : position(Cabana::slice<0>(aosoa)), + force(Cabana::slice<1>(aosoa)), + torque(Cabana::slice<2>(aosoa)), + charge(Cabana::slice<3>(aosoa)), + id(Cabana::slice<4>(aosoa)), + type(Cabana::slice<5>(aosoa)), + ghost(Cabana::slice<6>(aosoa)) {} +}; +#endif diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 23fd7e9ce1e..f6b1516b138 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -37,6 +37,7 @@ class CabanaData { ListType verlet_list; std::unordered_map id_to_index; std::vector index_to_id; + std::vector unique_particles; int particle_number; public: @@ -49,11 +50,14 @@ class CabanaData { : verlet_list(verlet_list), id_to_index(id_to_index) {} CabanaData(ListType verlet_list, int particle_number) : verlet_list(verlet_list), particle_number(particle_number) {} + CabanaData(ListType verlet_list, std::vector unique_particles, int particle_number) + : verlet_list(verlet_list), unique_particles(unique_particles), particle_number(particle_number) {} ListType get_verlet_list() const { return verlet_list; } std::unordered_map get_id_to_index() const { return id_to_index; } std::vector get_index_to_id() const { return index_to_id; } int get_index() const { return particle_number; } + std::vector get_unique_particles() const { return unique_particles; } ~CabanaData() {}; }; diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index bac43596562..8e37597ded9 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -30,8 +30,10 @@ #ifdef SHARED_MEMORY_PARALLELISM +#include "aosoa_pack.hpp" #include "cabana_data.hpp" #include "custom_verlet_list.hpp" +#include "verlet_list_loop.hpp" #include #include #include @@ -47,132 +49,25 @@ inline double wrap(double x, double L) { return result; } -template inline void write_particle(Particle const &p, int const &id, - SliceDouble3 &s_position, SliceDouble3 &s_force, - SliceDouble3 &s_torque, SliceDouble &s_charge, - SliceInt &s_id, SliceInt &s_type, SliceBool &s_ghost, - Utils::Vector3d &box_l) { + AoSoA_pack &aosoa, Utils::Vector3d &box_l) { auto const pos = p.pos(); - s_position(id, 0) = wrap(pos[0], box_l[0]); - s_position(id, 1) = wrap(pos[1], box_l[1]); - s_position(id, 2) = wrap(pos[2], box_l[2]); - s_id(id) = p.id(); - s_charge(id) = p.q(); - s_type(id) = p.type(); - s_ghost(id) = p.is_ghost(); - s_force(id, 0) = 0.0; - s_force(id, 1) = 0.0; - s_force(id, 2) = 0.0; - s_torque(id, 0) = 0.0; - s_torque(id, 1) = 0.0; - s_torque(id, 2) = 0.0; - assert(s_position(id, 0) >= 0. and s_position(id, 0) < box_l[0]); - assert(s_position(id, 1) >= 0. and s_position(id, 1) < box_l[1]); - assert(s_position(id, 2) >= 0. and s_position(id, 2) < box_l[2]); -} - -inline void set_offset_and_size_indexed_by_cid( - int &total_bins, int *cell_num, - Cabana::LinkedCellList &cell_list, - Kokkos::View &bin_offset, - Kokkos::View &bin_size) { - for (int cid = 0; cid < total_bins; ++cid) { - int dx[3] = {}; - dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); - dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / - cell_num[2]); - dx[2] = cid % cell_num[2]; - bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); - bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); - - // Calculate particle_bins - cell_list(cid); - } -} - -using ActiveProtocol = std::variant; -inline int set_interacting_pair_cell( - int &total_bins, int total_pair_cell, int *cell_num, int *delta_lebc, - int le_direction, int le_normal, - std::shared_ptr le_protocol, - Kokkos::View &bin_size, - Cabana::LinkedCellList &cell_list, - Kokkos::View &interacting_pair_cell) { - - constexpr int ijkIndexes[27][3] = { - {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, - {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, - {0, -1, 0}, {0, -1, 1}, {0, 0, -1}, {0, 0, 0}, {0, 0, 1}, - {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, - {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, - {1, 1, 0}, {1, 1, 1}}; - - int empty_pair_number = 0; - int pair_cell_id = 0; - for (int cid_i = 0; cid_i < total_bins; ++cid_i) { - // Obtaining 3 dimentional cell index from cid_i - int index[3] = {}; - cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); - int dx[3]; - // From 27 neighbor cell, the list of interacting pair cell is created - for (int n = 0; n < 27; ++n) { - bool duplicate_cell = false; - // Obtaining 3 dimentional cell index from neighbor cell - for (int d = 0; d < 3; ++d) { - dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; - if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) - duplicate_cell = true; - } - if (duplicate_cell) - continue; - - // Lees-Edwards BC - int le_crossing = 0; - if (le_protocol != nullptr) { - le_crossing = - ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } - } - // Additional Cell - /* - if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } - cell_offset = bin_offset(dx[0], dx[1], dx[2]); - cell_size = bin_size(dx[0], dx[1], dx[2]); - } - */ - - // Interacting pair cell is registered in the list - int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - if (cid_i <= cid_j) { - if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - ++pair_cell_id; - } else { - ++empty_pair_number; - } - } - } - } - return empty_pair_number; + aosoa.position(id, 0) = wrap(pos[0], box_l[0]); + aosoa.position(id, 1) = wrap(pos[1], box_l[1]); + aosoa.position(id, 2) = wrap(pos[2], box_l[2]); + aosoa.id(id) = p.id(); + aosoa.charge(id) = p.q(); + aosoa.type(id) = p.type(); + aosoa.ghost(id) = p.is_ghost(); + aosoa.force(id, 0) = 0.0; + aosoa.force(id, 1) = 0.0; + aosoa.force(id, 2) = 0.0; + aosoa.torque(id, 0) = 0.0; + aosoa.torque(id, 1) = 0.0; + aosoa.torque(id, 2) = 0.0; + assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); + assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); + assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); } template @@ -219,20 +114,15 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Setup"); #endif - // Dont know where to do this better - using data_types = Cabana::MemberTypes; - using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; - using execution_space = Kokkos::DefaultExecutionSpace; using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; // Number of threads - const int num_threads = execution_space().concurrency(); + int num_threads = execution_space().concurrency(); - const int vector_length = 1; + //const int vector_length = 1; #ifdef CALIPER CALI_MARK_END("Cabana - Setup"); #endif @@ -245,7 +135,8 @@ void cabana_short_range( #endif std::unordered_map id_to_index{}; // For DEBUG std::unordered_set registered_index{}; - std::vector index_to_id{}; + //std::vector index_to_id{}; + std::vector unique_particles; int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); @@ -263,30 +154,33 @@ void cabana_short_range( // map if (rebuild) { - for (auto const &p : particles) { - // if (cell_structure.get_local_particle(p.id())) { - id_to_index[p.id()] = index; + for (auto &p : particles) { + if (cell_structure.get_local_particle(p.id())) { + //id_to_index[p.id()] = index; registered_index.insert(p.id()); + unique_particles.emplace_back(&p); index++; - //} + } } - for (auto const &p : ghost_particles) { + for (auto &p : ghost_particles) { if (not registered_index.contains(p.id())) { - // if (cell_structure.get_local_particle(p.id())) { - id_to_index[p.id()] = index; + if (cell_structure.get_local_particle(p.id())) { + //id_to_index[p.id()] = index; registered_index.insert(p.id()); + unique_particles.emplace_back(&p); index++; - //} + } } } } else { // If we do not rebuild we can use the saved map // id_to_index = saved_data.get_id_to_index(); index = saved_data.get_index(); + unique_particles = saved_data.get_unique_particles(); } - const int number_of_unique_particles = index; + int number_of_unique_particles = index; #ifdef CALIPER CALI_MARK_END("Cabana - Index map"); #endif @@ -295,46 +189,22 @@ void cabana_short_range( // Create and fill particle storage // =================================================== #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Fill particle storage"); + CALI_MARK_BEGIN("Cabana - Allocation"); #endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); - auto slice_position = Cabana::slice<0>(particle_storage); - auto slice_force = Cabana::slice<1>(particle_storage); - auto slice_torque = Cabana::slice<2>(particle_storage); - auto slice_charge = Cabana::slice<3>(particle_storage); - auto slice_id = Cabana::slice<4>(particle_storage); - auto slice_type = Cabana::slice<5>(particle_storage); - auto slice_ghost = Cabana::slice<6>(particle_storage); + auto aosoa = AoSoA_pack(particle_storage); // particle properties are defined in aosoa_pack.hpp auto box_l = box_geo.length(); - int p_id = 0; - registered_index.clear(); - for (auto const &p : particles) { - // if (!cell_structure.get_local_particle(p.id())) continue; - write_particle(p, p_id, slice_position, slice_force, slice_torque, - slice_charge, slice_id, slice_type, slice_ghost, box_l); - registered_index.insert(p.id()); - ++p_id; - } - for (auto const &p : ghost_particles) { - // if the ghost is not in the previous map, but mpi moved it to this rank? - // it will not have neighbors because we did not rebuild the verlet list. - if (registered_index.contains(p.id())) { - continue; - } - // if (!cell_structure.get_local_particle(p.id())) continue; - write_particle(p, p_id, slice_position, slice_force, slice_torque, - slice_charge, slice_id, slice_type, slice_ghost, box_l); - registered_index.insert(p.id()); - ++p_id; - } - - using TP = decltype(slice_position); - // using TF = decltype(slice_force); - // using TR = decltype(slice_torque); - using TQ = decltype(slice_charge); - using TI = decltype(slice_id); - using TT = decltype(slice_type); + //int p_id = 0; + //registered_index.clear(); + //Kokkos::View particle_view("particle_pointer", unique_particles.size()); + Kokkos::RangePolicy allocation_policy(0, unique_particles.size()); + Kokkos::parallel_for("allocation", allocation_policy, [&](int p_id) { + auto p = *unique_particles[p_id]; + //if (!cell_structure.get_local_particle(p.id())) continue; + write_particle(p, p_id, aosoa, box_l); + }); + Kokkos::fence(); Kokkos::View local_force( "local_force", num_threads, number_of_unique_particles, 3); @@ -346,72 +216,90 @@ void cabana_short_range( num_threads, 3); #ifdef CALIPER - CALI_MARK_END("Cabana - Fill particle storage"); + CALI_MARK_END("Cabana - Allocation"); #endif // The kernel of calculate force struct FirstNeighborKernel { - const CellStructure *cell; +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + std::vector unique_particles; +#endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias; const InteractionsNonBonded &nonbonded_ias; - const Thermostat::Thermostat &thermostat; const BoxGeometry &box_geo; // std::vector &index_to_id; Kokkos::View local_force; Kokkos::View local_torque; Kokkos::View local_virial; - TP &slice_position; - TQ &slice_charge; - TI &slice_id; - TT &slice_type; + AoSoA_pack aosoa; #ifdef COLLISION_DETECTION // std::shared_ptr // collision_detection; mutable CollisionDetection::CollisionDetection collision_detection; #endif Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; + const Thermostat::Thermostat &thermostat; +#endif int num_threads; int mpi_rank; int particle_number; FirstNeighborKernel( - const CellStructure *cell_, + //const CellStructure *cell_, +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + std::vector &unique_particles_, +#endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, - const Thermostat::Thermostat &thermostat_, const BoxGeometry &box_geo_, // std::vector &index_to_id_, Kokkos::View local_force_, Kokkos::View local_torque_, - Kokkos::View local_virial_, TP &slice_position_, - TQ &slice_charge_, TI &slice_id_, TT &slice_type_, + Kokkos::View local_virial_, + AoSoA_pack &aosoa_, #ifdef COLLISION_DETECTION // std::shared_ptr // collision_detection_, CollisionDetection::CollisionDetection collision_detection_, #endif Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, - int num_threads_, int mpi_rank_, int particle_number_) - : cell(cell_), bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - thermostat(thermostat_), box_geo(box_geo_), + const Thermostat::Thermostat &thermostat_, +#endif + int &num_threads_, int &mpi_rank_, int &particle_number_) + : //cell(cell_), +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles(unique_particles_), +#endif + bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + box_geo(box_geo_), // index_to_id(index_to_id_), local_force(local_force_), local_torque(local_torque_), - local_virial(local_virial_), slice_position(slice_position_), - slice_charge(slice_charge_), slice_id(slice_id_), - slice_type(slice_type_), + local_virial(local_virial_), aosoa(aosoa_), #ifdef COLLISION_DETECTION collision_detection(collision_detection_), #endif - coulomb_kernel(coulomb_kernel_), dipoles_kernel(dipoles_kernel_), + coulomb_kernel(coulomb_kernel_), +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), + thermostat(thermostat_), +#endif num_threads(num_threads_), mpi_rank(mpi_rank_), particle_number(particle_number_) { } @@ -419,48 +307,54 @@ void cabana_short_range( KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { - Utils::Vector3d const pi = {slice_position(i, 0), slice_position(i, 1), - slice_position(i, 2)}; - Utils::Vector3d const pj = {slice_position(j, 0), slice_position(j, 1), - slice_position(j, 2)}; - - Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); - auto const dist = d.norm(); - - auto const q1q2 = slice_charge(i) * slice_charge(j); - auto thread_id = omp_get_thread_num(); // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); // std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << // " " << - + IA_parameters const &ia_params = - nonbonded_ias.get_ia_param(slice_type(i), slice_type(j)); + nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); /* - auto p1 = cell->get_local_particle(slice_id(i)); - auto p2 = cell->get_local_particle(slice_id(j)); + auto p1 = unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); + auto p2 = unique_particles.at(j);// cell->get_local_particle(aosoa.id(j)); + + Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), p2->pos()); + auto const dist = d.norm(); - // if (p1 == nullptr or p2 == nullptr) - // return; + auto const q1q2 =aosoa.charge(i) *aosoa.charge(j); auto const dist2 = dist * dist; - auto [pf, virial] = add_non_bonded_pair_force( +#ifdef NPT + auto [pf, virial] +#else + auto pf +#endif + = add_non_bonded_pair_force( const_cast(*p1), const_cast(*p2), d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); */ - + ParticleForce pf{}; #ifdef NPT Utils::Vector3d virial{}; #endif + Utils::Vector3d const pi = {aosoa.position(i, 0),aosoa.position(i, 1), + aosoa.position(i, 2)}; + Utils::Vector3d const pj = {aosoa.position(j, 0),aosoa.position(j, 1), + aosoa.position(j, 2)}; + + Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); + auto const dist = d.norm(); + + auto const q1q2 =aosoa.charge(i) *aosoa.charge(j); #ifdef EXCLUSIONS - auto p1 = cell->get_local_particle(slice_id(i)); - auto p2 = cell->get_local_particle(slice_id(j)); + auto p1 = unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); + auto p2 = unique_particles.at(j);// cell->get_local_particle(aosoa.id(j)); - if (p1 == nullptr or p2 == nullptr) - return; + //if (p1 == nullptr or p2 == nullptr) + // return; bool do_nonbonded_flag = do_nonbonded(*p1, *p2); #else @@ -475,11 +369,11 @@ void cabana_short_range( auto const dist2 = dist * dist; #ifndef EXCLUSIONS - auto p1 = cell->get_local_particle(slice_id(i)); - auto p2 = cell->get_local_particle(slice_id(j)); + auto p1 = unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); + auto p2 = unique_particles.at(j);// cell->get_local_particle(aosoa.id(j)); - if (p1 == nullptr or p2 == nullptr) - return; + //if (p1 == nullptr or p2 == nullptr) + // return; #endif // NOT EXCLUSIONS add_non_bonded_pair_force_with_p( const_cast(*p1), const_cast(*p2), pf, @@ -494,18 +388,21 @@ void cabana_short_range( local_force(thread_id, i, 0) += pf.f[0]; local_force(thread_id, i, 1) += pf.f[1]; local_force(thread_id, i, 2) += pf.f[2]; +#ifdef ROTATION local_torque(thread_id, i, 0) += pf.torque[0]; local_torque(thread_id, i, 1) += pf.torque[1]; local_torque(thread_id, i, 2) += pf.torque[2]; +#endif auto opf = calc_opposing_force(pf, d); local_force(thread_id, j, 0) += opf.f[0]; local_force(thread_id, j, 1) += opf.f[1]; local_force(thread_id, j, 2) += opf.f[2]; +#ifdef ROTATION local_torque(thread_id, j, 0) += opf.torque[0]; local_torque(thread_id, j, 1) += opf.torque[1]; local_torque(thread_id, j, 2) += opf.torque[2]; - +#endif #ifdef NPT local_virial(thread_id, 0) += virial[0]; local_virial(thread_id, 1) += virial[1]; @@ -530,7 +427,7 @@ void cabana_short_range( ListType verlet_list; // Rebuild verlet list if needed - auto const &system = ::System::get_system(); + //auto const &system = ::System::get_system(); int max_counts; double max_cutoff = pair_cutoff; // system.get_interaction_range(); if (std::isinf(max_cutoff)) { @@ -543,7 +440,7 @@ void cabana_short_range( max_counts = 256; if (rebuild) { // Legacy Velert List /*verlet_list = - ListType(slice_position, 0, slice_position.size(), max_counts); + ListType(aosoa.position, 0,aosoa.position.size(), max_counts); auto kernel = [&](Particle const &p1, Particle const &p2) { verlet_list.addNeighbor(id_to_index.at(p1.id()), id_to_index.at(p2.id())); @@ -566,187 +463,76 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List by ESPRESSO"); #endif -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List and calc Force"); -#endif FirstNeighborKernel first_neighbor_kernel( - &cell_structure, bonded_ias, nonbonded_ias, thermostat, box_geo, - // index_to_id, - local_force, local_torque, local_virial, slice_position, slice_charge, - slice_id, slice_type, +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles, +#endif + bonded_ias, nonbonded_ias, box_geo, + local_force, local_torque, local_virial, + aosoa, #ifdef COLLISION_DETECTION *collision_detection, #endif - coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel, + coulomb_kernel, +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, +#endif num_threads, rank, number_of_unique_particles); if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory - // Creating LinkedCellList and VerletList: - // Box Properties - Cabana::LinkedCellList cell_list; - double grid_min[3] = {0.0, 0.0, 0.0}; - double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; - double grid_delta[3] = {}; - int cell_num[3] = {}; - double eff_cutoff; - for (int d = 0; d < 3; ++d) { - eff_cutoff = max_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); - } - // For Lees-Edwards boundary condition - double le_offset; - int le_direction; - int le_normal; - int delta_lebc[3] = {0, 0, 0}; - auto le_protocol = system.lees_edwards->get_protocol(); - if (le_protocol == nullptr) { - le_offset = 0.; - le_direction = -1; - le_normal = -1; - } else { - le_offset = box_geo.lees_edwards_bc().pos_offset; - le_direction = box_geo.lees_edwards_bc().shear_direction; - le_normal = box_geo.lees_edwards_bc().shear_plane_normal; - delta_lebc[le_direction] = - static_cast(std::ceil(le_offset / grid_delta[le_direction])) % - cell_num[le_direction]; - } - cell_list = Cabana::createLinkedCellList( - slice_position, grid_delta, grid_min, grid_max); - int total_bins = cell_list.totalBins(); - // Now permute the AoSoA (i.e. reorder the data) using the linked cell - // list. - // Cabana::permute( cell_list, particle_storage ); - - verlet_list = - ListType(slice_position, 0, slice_position.size(), max_counts); - - // Offset particle id and the number of particle in specific cell - Kokkos::View bin_offset("bin_offset", - total_bins); - Kokkos::View bin_size("bin_size", total_bins); - set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, - bin_offset, bin_size); - auto const particle_bins = cell_list.getParticleBins(); - - // Creating Interacting cell - int total_pair_cell; - if (total_bins < 27) { - total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; - } else { - total_pair_cell = 14 * total_bins; - } - Kokkos::View interacting_pair_cell( - "interacting_pair_cell", total_pair_cell, 2); - int empty_pair_number = set_interacting_pair_cell( - total_bins, total_pair_cell, cell_num, delta_lebc, le_direction, - le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); - - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - - // This kernel used the loop for the pair of interacting cell - auto kernel = [&](const int pair_cell_i) { - int cid_i = interacting_pair_cell(pair_cell_i, 0); - int cid_j = interacting_pair_cell(pair_cell_i, 1); - - auto verlet_kernel = [&](Particle *p1, int ii, int id_i, - int cell_offset, int cell_size) { - for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - // int ii = cell_list.permutation(i); // debug - // int jj = j; - int jj = cell_list.permutation(j); - int id_j = slice_id(jj); - if (slice_ghost(ii) or slice_ghost(jj)) { - if ((id_i < id_j and slice_ghost(ii)) or - (id_i > id_j and slice_ghost(jj))) { - continue; - } - } else if (slice_ghost(ii) and slice_ghost(jj)) { - continue; // reject both ghost - } - auto p2 = cell_structure.get_local_particle(id_j); - if (p2 == nullptr) - continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " - << slice_ghost(ii) << " " - << slice_ghost(jj) << " " - << cid_i << " " - << cid_j << " " - << slice_position(ii, 0) << ", " - << slice_position(ii, 1) << ", " - << slice_position(ii, 2) << " " - << slice_position(jj, 0) << ", " - << slice_position(jj, 1) << ", " - << slice_position(jj, 2) << "\n";// - //std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ - first_neighbor_kernel(ii, jj); - } - } // j-loop - }; - - int offset_i = bin_offset(cid_i); - int size_i = bin_size(cid_i); - - for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; - int ii = cell_list.permutation(i); - int id_i = slice_id(ii); - auto p1 = cell_structure.get_local_particle(id_i); - if (p1 == nullptr) - continue; - - if (cid_i == cid_j) { - verlet_kernel(p1, ii, id_i, i + 1, - size_i + offset_i - i - 1); // j-loop - // verlet_kernel(p1, i, id_i, i + 1, - // size_i + offset_i - i - 1); // j-loop - } else { - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop - // verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop - } - } // i-loop - }; - - Kokkos::RangePolicy policy(0, total_pair_cell - - empty_pair_number); - Kokkos::parallel_for("calc_by_cell_list", policy, kernel); - Kokkos::fence(); +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); +#endif + verlet_list = create_verlet_list( + max_cutoff, max_counts, aosoa, + unique_particles, verlet_criterion, first_neighbor_kernel, cell_structure); +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List by Cabana"); +#endif } else { - //{ + //{ +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - calc Force"); +#endif + /* + using neighbor_list = Cabana::NeighborList; + std::vector> interaction_pairs; + + for (int i = 0; i < number_of_unique_particles; ++i) { + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + interaction_pairs.emplace_back(i, j); + } + } + */ + /* + Kokkos::parallel_for("ForceLoop", Kokkos::RangePolicy<>(0, interaction_pairs.size()), + KOKKOS_LAMBDA(int idx) { + auto i = interaction_pairs[idx].first; + auto j = interaction_pairs[idx].second; + first_neighbor_kernel(i, j); + }); + */ + Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), Cabana::TeamOpTag()); + Kokkos::fence(); +#ifdef CALIPER + CALI_MARK_END("Cabana - calc Force"); +#endif } // Save data for next iteration if we just rebuilt if (rebuild) { - CabanaData new_data(verlet_list, particle_storage.size()); + CabanaData new_data(verlet_list, unique_particles, particle_storage.size()); cell_structure.set_cabana_data(std::make_unique(new_data)); } -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List and calc Force"); -#endif #ifdef CALIPER CALI_MARK_BEGIN("Cabana - reduction Forces"); @@ -769,12 +555,12 @@ void cabana_short_range( ty += local_torque(tid, i, 1); tz += local_torque(tid, i, 2); } - slice_force(i, 0) = fx; - slice_force(i, 1) = fy; - slice_force(i, 2) = fz; - slice_torque(i, 0) = tx; - slice_torque(i, 1) = ty; - slice_torque(i, 2) = tz; + aosoa.force(i, 0) = fx; + aosoa.force(i, 1) = fy; + aosoa.force(i, 2) = fz; + aosoa.torque(i, 0) = tx; + aosoa.torque(i, 1) = ty; + aosoa.torque(i, 2) = tz; }); Kokkos::fence(); @@ -817,16 +603,20 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - Particle Forces"); #endif for (auto id = 0; id < particle_storage.size(); ++id) { - auto p = cell_structure.get_local_particle(slice_id(id)); + auto p = cell_structure.get_local_particle(aosoa.id(id)); if (p == nullptr) { return; } - Utils::Vector3d f_vec{slice_force(id, 0), slice_force(id, 1), - slice_force(id, 2)}; - Utils::Vector3d torque_vec{slice_torque(id, 0), slice_torque(id, 1), - slice_torque(id, 2)}; + Utils::Vector3d f_vec{aosoa.force(id, 0), aosoa.force(id, 1), + aosoa.force(id, 2)}; + Utils::Vector3d torque_vec{aosoa.torque(id, 0), aosoa.torque(id, 1), + aosoa.torque(id, 2)}; +#ifdef ROTATION ParticleForce f(f_vec, torque_vec); +#else + ParticleForce f(f_vec); +#endif p->force_and_torque() += f; } #ifdef CALIPER diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp new file mode 100644 index 00000000000..94ecb3c8aac --- /dev/null +++ b/src/core/verlet_list_loop.hpp @@ -0,0 +1,317 @@ +/* + * Copyright (C) 2010-2025 The ESPResSo project + * + * This file is part of ESPResSo. + * + * ESPResSo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ESPResSo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "config/config.hpp" + +#include "cell_system/CellStructure.hpp" +#include "lees_edwards/lees_edwards.hpp" + +#ifdef CALIPER +#include +#endif + +#ifdef SHARED_MEMORY_PARALLELISM + +#include "aosoa_pack.hpp" +#include "cabana_data.hpp" +#include "custom_verlet_list.hpp" +#include +#include +#include +#include +#include +#include +#include + +using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; +using execution_space = Kokkos::DefaultExecutionSpace; + +inline void set_offset_and_size_indexed_by_cid( + int &total_bins, int *cell_num, + Cabana::LinkedCellList &cell_list, + Kokkos::View &bin_offset, + Kokkos::View &bin_size) { + for (int cid = 0; cid < total_bins; ++cid) { + int dx[3] = {}; + dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); + dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / + cell_num[2]); + dx[2] = cid % cell_num[2]; + bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); + bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); + + // Calculate particle_bins + cell_list(cid); + } +} + +using ActiveProtocol = std::variant; +inline int set_interacting_pair_cell( + int &total_bins, int total_pair_cell, int *cell_num, int *delta_lebc, + int le_direction, int le_normal, + std::shared_ptr le_protocol, + //ActiveProtocol le_protocol, + Kokkos::View &bin_size, + Cabana::LinkedCellList &cell_list, + Kokkos::View &interacting_pair_cell) { + + constexpr int ijkIndexes[27][3] = { + {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, + {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, + {0, -1, 0}, {0, -1, 1}, {0, 0, -1}, {0, 0, 0}, {0, 0, 1}, + {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, + {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, + {1, 1, 0}, {1, 1, 1}}; + + int empty_pair_number = 0; + int pair_cell_id = 0; + for (int cid_i = 0; cid_i < total_bins; ++cid_i) { + // Obtaining 3 dimentional cell index from cid_i + int index[3] = {}; + cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); + int dx[3]; + // From 27 neighbor cell, the list of interacting pair cell is created + for (int n = 0; n < 27; ++n) { + bool duplicate_cell = false; + // Obtaining 3 dimentional cell index from neighbor cell + for (int d = 0; d < 3; ++d) { + dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; + if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) + duplicate_cell = true; + } + if (duplicate_cell) + continue; + + // Lees-Edwards BC + int le_crossing = 0; + if (le_protocol != nullptr) { + le_crossing = + ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } + } + // Additional Cell + /* + if (le_crossing != 0 && index[le_direction] == 1) { + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } + cell_offset = bin_offset(dx[0], dx[1], dx[2]); + cell_size = bin_size(dx[0], dx[1], dx[2]); + } + */ + + // Interacting pair cell is registered in the list + int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); + if (cid_i <= cid_j) { + if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } else { + ++empty_pair_number; + } + } + } + } + return empty_pair_number; +} + +using ListAlgorithm = Cabana::HalfNeighborTag; +using ListType = Cabana::CustomVerletList; +//template +template +ListType create_verlet_list( + double const max_cutoff, int const max_counts, + AoSoA_pack aosoa, + std::vector unique_particles, + VerletCriterion const &verlet_criterion, + Kernel first_neighbor_kernel, + CellStructure &cell_structure) { + // Creating LinkedCellList and VerletList: + // Box Properties + auto const &system = ::System::get_system(); + auto box_geo = *(system.box_geo); + auto box_l = box_geo.length(); + Cabana::LinkedCellList cell_list; + double grid_min[3] = {0.0, 0.0, 0.0}; + double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; + double grid_delta[3] = {}; + int cell_num[3] = {}; + double eff_cutoff; + for (int d = 0; d < 3; ++d) { + eff_cutoff = max_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + } + // For Lees-Edwards boundary condition + double le_offset; + int le_direction; + int le_normal; + int delta_lebc[3] = {0, 0, 0}; + auto le_protocol = system.lees_edwards->get_protocol(); + //std::shared_ptr le_protocol = system.lees_edwards->get_protocol(); + if (le_protocol == nullptr) { + le_offset = 0.; + le_direction = -1; + le_normal = -1; + } else { + le_offset = box_geo.lees_edwards_bc().pos_offset; + le_direction = box_geo.lees_edwards_bc().shear_direction; + le_normal = box_geo.lees_edwards_bc().shear_plane_normal; + delta_lebc[le_direction] = + static_cast(std::ceil(le_offset / grid_delta[le_direction])) % + cell_num[le_direction]; + } + cell_list = Cabana::createLinkedCellList( + aosoa.position, grid_delta, grid_min, grid_max); + int total_bins = cell_list.totalBins(); + // Now permute the AoSoA (i.e. reorder the data) using the linked cell + // list. + // Cabana::permute( cell_list, particle_storage ); + + ListType verlet_list = + ListType(aosoa.position, 0, aosoa.position.size(), max_counts); + + // Offset particle id and the number of particle in specific cell + Kokkos::View bin_offset("bin_offset", + total_bins); + Kokkos::View bin_size("bin_size", total_bins); + set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, + bin_offset, bin_size); + auto const particle_bins = cell_list.getParticleBins(); + + // Creating Interacting cell + int total_pair_cell; + if (total_bins < 27) { + total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; + } else { + total_pair_cell = 14 * total_bins; + } + Kokkos::View interacting_pair_cell( + "interacting_pair_cell", total_pair_cell, 2); + int empty_pair_number = set_interacting_pair_cell( + total_bins, total_pair_cell, cell_num, delta_lebc, le_direction, + le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); + + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; + + // This kernel used the loop for the pair of interacting cell + auto kernel = [&](const int pair_cell_i) { + int cid_i = interacting_pair_cell(pair_cell_i, 0); + int cid_j = interacting_pair_cell(pair_cell_i, 1); + + auto verlet_kernel = [&](Particle *p1, int ii, int id_i, + int cell_offset, int cell_size) { + for (int j = cell_offset; j < cell_offset + cell_size; ++j) { + //int ii = cell_list.permutation(i); // debug + // int jj = j; + int jj = cell_list.permutation(j); + int id_j = aosoa.id(jj); + if (aosoa.ghost(ii) or aosoa.ghost(jj)) { + if (((id_i < id_j) and aosoa.ghost(ii)) or + ((id_i > id_j) and aosoa.ghost(jj))) { + continue; + } + } else if (aosoa.ghost(ii) and aosoa.ghost(jj)) { + continue; // reject both ghost + } + auto p2 = unique_particles.at(jj);//cell_structure.get_local_particle(id_j); + //if (p2 == nullptr) + // continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << aosoa.ghost(ii) << " " + << aosoa.ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << aosoa.position(ii, 0) << ", " + << aosoa.position(ii, 1) << ", " + << aosoa.position(ii, 2) << " " + << aosoa.position(jj, 0) << ", " + << aosoa.position(jj, 1) << ", " + << aosoa.position(jj, 2) << "\n";// + //std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + first_neighbor_kernel(ii, jj); + } + } // j-loop + }; + + int offset_i = bin_offset(cid_i); + int size_i = bin_size(cid_i); + + for (int i = offset_i; i < offset_i + size_i; ++i) { + // int ii = i; + int ii = cell_list.permutation(i); //get previous id + int id_i = aosoa.id(ii); + auto p1 = unique_particles.at(ii); //cell_structure.get_local_particle(id_i); + //if (p1 == nullptr) + // continue; + + if (cid_i == cid_j) { + verlet_kernel(p1, ii, id_i, i + 1, + size_i + offset_i - i - 1); // j-loop + // verlet_kernel(p1, i, id_i, i + 1, + // size_i + offset_i - i - 1); // j-loop + } else { + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); + verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop + //verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop + } + } // i-loop + }; + + Kokkos::RangePolicy policy(0, total_pair_cell - + empty_pair_number); + Kokkos::parallel_for("calc_by_cell_list", policy, kernel); + Kokkos::fence(); + + return verlet_list; +} +#endif // SHARED_MEMORY_PARALLELISM diff --git a/testsuite/python/exclusions.py b/testsuite/python/exclusions.py index 17887ee208d..3267a0a95f5 100644 --- a/testsuite/python/exclusions.py +++ b/testsuite/python/exclusions.py @@ -59,7 +59,6 @@ def test_transfer(self): i = 0 for _ in range(15): - print(i) i += 1 self.system.integrator.run(100) self.assertEqual(list(p0.exclusions), [1, 2, 3]) From 01cc1d8872f4968fc38cc133ef9a7e8860931f43 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 20 Jun 2025 08:22:53 +0200 Subject: [PATCH 35/94] Formatting --- src/core/aosoa_pack.hpp | 33 +++-- src/core/cabana_data.hpp | 12 +- src/core/short_range_cabana.hpp | 189 ++++++++++++++-------------- src/core/verlet_list_loop.hpp | 214 ++++++++++++++++---------------- 4 files changed, 228 insertions(+), 220 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index 9a88ff02049..01341a7d473 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -24,30 +24,27 @@ #include const int vector_length = 1; -using data_types = Cabana::MemberTypes; +using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using AoSoA_type = Cabana::AoSoA; struct AoSoA_pack { - AoSoA_type::member_slice_type<0> position; - AoSoA_type::member_slice_type<1> force; - AoSoA_type::member_slice_type<2> torque; - AoSoA_type::member_slice_type<3> charge; - AoSoA_type::member_slice_type<4> id; - AoSoA_type::member_slice_type<5> type; - AoSoA_type::member_slice_type<6> ghost; + AoSoA_type::member_slice_type<0> position; + AoSoA_type::member_slice_type<1> force; + AoSoA_type::member_slice_type<2> torque; + AoSoA_type::member_slice_type<3> charge; + AoSoA_type::member_slice_type<4> id; + AoSoA_type::member_slice_type<5> type; + AoSoA_type::member_slice_type<6> ghost; - AoSoA_pack() = default; + AoSoA_pack() = default; - AoSoA_pack(AoSoA_type &aosoa) - : position(Cabana::slice<0>(aosoa)), - force(Cabana::slice<1>(aosoa)), - torque(Cabana::slice<2>(aosoa)), - charge(Cabana::slice<3>(aosoa)), - id(Cabana::slice<4>(aosoa)), - type(Cabana::slice<5>(aosoa)), - ghost(Cabana::slice<6>(aosoa)) {} + AoSoA_pack(AoSoA_type &aosoa) + : position(Cabana::slice<0>(aosoa)), force(Cabana::slice<1>(aosoa)), + torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), + id(Cabana::slice<4>(aosoa)), type(Cabana::slice<5>(aosoa)), + ghost(Cabana::slice<6>(aosoa)) {} }; #endif diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index f6b1516b138..7b6028e8b7e 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -37,7 +37,7 @@ class CabanaData { ListType verlet_list; std::unordered_map id_to_index; std::vector index_to_id; - std::vector unique_particles; + std::vector unique_particles; int particle_number; public: @@ -50,14 +50,18 @@ class CabanaData { : verlet_list(verlet_list), id_to_index(id_to_index) {} CabanaData(ListType verlet_list, int particle_number) : verlet_list(verlet_list), particle_number(particle_number) {} - CabanaData(ListType verlet_list, std::vector unique_particles, int particle_number) - : verlet_list(verlet_list), unique_particles(unique_particles), particle_number(particle_number) {} + CabanaData(ListType verlet_list, std::vector unique_particles, + int particle_number) + : verlet_list(verlet_list), unique_particles(unique_particles), + particle_number(particle_number) {} ListType get_verlet_list() const { return verlet_list; } std::unordered_map get_id_to_index() const { return id_to_index; } std::vector get_index_to_id() const { return index_to_id; } int get_index() const { return particle_number; } - std::vector get_unique_particles() const { return unique_particles; } + std::vector get_unique_particles() const { + return unique_particles; + } ~CabanaData() {}; }; diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 8e37597ded9..9a52b2ac177 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -49,8 +49,8 @@ inline double wrap(double x, double L) { return result; } -inline void write_particle(Particle const &p, int const &id, - AoSoA_pack &aosoa, Utils::Vector3d &box_l) { +inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, + Utils::Vector3d &box_l) { auto const pos = p.pos(); aosoa.position(id, 0) = wrap(pos[0], box_l[0]); aosoa.position(id, 1) = wrap(pos[1], box_l[1]); @@ -122,7 +122,7 @@ void cabana_short_range( // Number of threads int num_threads = execution_space().concurrency(); - //const int vector_length = 1; + // const int vector_length = 1; #ifdef CALIPER CALI_MARK_END("Cabana - Setup"); #endif @@ -135,8 +135,8 @@ void cabana_short_range( #endif std::unordered_map id_to_index{}; // For DEBUG std::unordered_set registered_index{}; - //std::vector index_to_id{}; - std::vector unique_particles; + // std::vector index_to_id{}; + std::vector unique_particles; int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); @@ -156,20 +156,20 @@ void cabana_short_range( for (auto &p : particles) { if (cell_structure.get_local_particle(p.id())) { - //id_to_index[p.id()] = index; - registered_index.insert(p.id()); - unique_particles.emplace_back(&p); - index++; + // id_to_index[p.id()] = index; + registered_index.insert(p.id()); + unique_particles.emplace_back(&p); + index++; } } for (auto &p : ghost_particles) { if (not registered_index.contains(p.id())) { if (cell_structure.get_local_particle(p.id())) { - //id_to_index[p.id()] = index; - registered_index.insert(p.id()); - unique_particles.emplace_back(&p); - index++; + // id_to_index[p.id()] = index; + registered_index.insert(p.id()); + unique_particles.emplace_back(&p); + index++; } } } @@ -193,17 +193,20 @@ void cabana_short_range( #endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); - auto aosoa = AoSoA_pack(particle_storage); // particle properties are defined in aosoa_pack.hpp + auto aosoa = AoSoA_pack( + particle_storage); // particle properties are defined in aosoa_pack.hpp auto box_l = box_geo.length(); - //int p_id = 0; - //registered_index.clear(); - //Kokkos::View particle_view("particle_pointer", unique_particles.size()); - Kokkos::RangePolicy allocation_policy(0, unique_particles.size()); + // int p_id = 0; + // registered_index.clear(); + // Kokkos::View particle_view("particle_pointer", + // unique_particles.size()); + Kokkos::RangePolicy allocation_policy( + 0, unique_particles.size()); Kokkos::parallel_for("allocation", allocation_policy, [&](int p_id) { - auto p = *unique_particles[p_id]; - //if (!cell_structure.get_local_particle(p.id())) continue; - write_particle(p, p_id, aosoa, box_l); - }); + auto p = *unique_particles[p_id]; + // if (!cell_structure.get_local_particle(p.id())) continue; + write_particle(p, p_id, aosoa, box_l); + }); Kokkos::fence(); Kokkos::View local_force( @@ -221,9 +224,9 @@ void cabana_short_range( // The kernel of calculate force struct FirstNeighborKernel { -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - std::vector unique_particles; +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + std::vector unique_particles; #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias; const InteractionsNonBonded &nonbonded_ias; @@ -252,10 +255,10 @@ void cabana_short_range( int particle_number; FirstNeighborKernel( - //const CellStructure *cell_, -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - std::vector &unique_particles_, + // const CellStructure *cell_, +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + std::vector &unique_particles_, #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, @@ -263,8 +266,7 @@ void cabana_short_range( // std::vector &index_to_id_, Kokkos::View local_force_, Kokkos::View local_torque_, - Kokkos::View local_virial_, - AoSoA_pack &aosoa_, + Kokkos::View local_virial_, AoSoA_pack &aosoa_, #ifdef COLLISION_DETECTION // std::shared_ptr // collision_detection_, @@ -280,12 +282,12 @@ void cabana_short_range( const Thermostat::Thermostat &thermostat_, #endif int &num_threads_, int &mpi_rank_, int &particle_number_) - : //cell(cell_), -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles(unique_particles_), + : // cell(cell_), +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles(unique_particles_), #endif - bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), box_geo(box_geo_), // index_to_id(index_to_id_), local_force(local_force_), local_torque(local_torque_), @@ -296,9 +298,8 @@ void cabana_short_range( coulomb_kernel(coulomb_kernel_), #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel(dipoles_kernel_), - elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), - thermostat(thermostat_), + dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), + coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), #endif num_threads(num_threads_), mpi_rank(mpi_rank_), particle_number(particle_number_) { @@ -311,12 +312,13 @@ void cabana_short_range( // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); // std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << // " " << - + IA_parameters const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); /* - auto p1 = unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); - auto p2 = unique_particles.at(j);// cell->get_local_particle(aosoa.id(j)); + auto p1 = unique_particles.at(i); // +cell->get_local_particle(aosoa.id(i)); auto p2 = unique_particles.at(j);// +cell->get_local_particle(aosoa.id(j)); Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), p2->pos()); auto const dist = d.norm(); @@ -327,34 +329,36 @@ void cabana_short_range( #ifdef NPT auto [pf, virial] #else - auto pf + auto pf #endif - = add_non_bonded_pair_force( + = add_non_bonded_pair_force( const_cast(*p1), const_cast(*p2), d, dist, dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); */ - + ParticleForce pf{}; #ifdef NPT Utils::Vector3d virial{}; #endif - Utils::Vector3d const pi = {aosoa.position(i, 0),aosoa.position(i, 1), + Utils::Vector3d const pi = {aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2)}; - Utils::Vector3d const pj = {aosoa.position(j, 0),aosoa.position(j, 1), + Utils::Vector3d const pj = {aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)}; Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); auto const dist = d.norm(); - auto const q1q2 =aosoa.charge(i) *aosoa.charge(j); + auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); #ifdef EXCLUSIONS - auto p1 = unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); - auto p2 = unique_particles.at(j);// cell->get_local_particle(aosoa.id(j)); + auto p1 = + unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); + auto p2 = + unique_particles.at(j); // cell->get_local_particle(aosoa.id(j)); - //if (p1 == nullptr or p2 == nullptr) - // return; + // if (p1 == nullptr or p2 == nullptr) + // return; bool do_nonbonded_flag = do_nonbonded(*p1, *p2); #else @@ -369,11 +373,13 @@ void cabana_short_range( auto const dist2 = dist * dist; #ifndef EXCLUSIONS - auto p1 = unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); - auto p2 = unique_particles.at(j);// cell->get_local_particle(aosoa.id(j)); + auto p1 = + unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); + auto p2 = + unique_particles.at(j); // cell->get_local_particle(aosoa.id(j)); - //if (p1 == nullptr or p2 == nullptr) - // return; + // if (p1 == nullptr or p2 == nullptr) + // return; #endif // NOT EXCLUSIONS add_non_bonded_pair_force_with_p( const_cast(*p1), const_cast(*p2), pf, @@ -427,7 +433,7 @@ void cabana_short_range( ListType verlet_list; // Rebuild verlet list if needed - //auto const &system = ::System::get_system(); + // auto const &system = ::System::get_system(); int max_counts; double max_cutoff = pair_cutoff; // system.get_interaction_range(); if (std::isinf(max_cutoff)) { @@ -465,72 +471,71 @@ void cabana_short_range( #endif FirstNeighborKernel first_neighbor_kernel( -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles, +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles, #endif - bonded_ias, nonbonded_ias, box_geo, - local_force, local_torque, local_virial, - aosoa, + bonded_ias, nonbonded_ias, box_geo, local_force, local_torque, + local_virial, aosoa, #ifdef COLLISION_DETECTION *collision_detection, #endif coulomb_kernel, #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, + dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, #endif num_threads, rank, number_of_unique_particles); if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); + CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); #endif - verlet_list = create_verlet_list( - max_cutoff, max_counts, aosoa, - unique_particles, verlet_criterion, first_neighbor_kernel, cell_structure); + verlet_list = create_verlet_list(max_cutoff, max_counts, aosoa, + unique_particles, verlet_criterion, + first_neighbor_kernel, cell_structure); #ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List by Cabana"); + CALI_MARK_END("Cabana - Verlet List by Cabana"); #endif } else { - //{ + //{ #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - calc Force"); + CALI_MARK_BEGIN("Cabana - calc Force"); #endif - /* + /* using neighbor_list = Cabana::NeighborList; std::vector> interaction_pairs; for (int i = 0; i < number_of_unique_particles; ++i) { - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - interaction_pairs.emplace_back(i, j); - } + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + interaction_pairs.emplace_back(i, j); + } } */ /* - Kokkos::parallel_for("ForceLoop", Kokkos::RangePolicy<>(0, interaction_pairs.size()), - KOKKOS_LAMBDA(int idx) { - auto i = interaction_pairs[idx].first; - auto j = interaction_pairs[idx].second; - first_neighbor_kernel(i, j); - }); + Kokkos::parallel_for("ForceLoop", Kokkos::RangePolicy<>(0, + interaction_pairs.size()), KOKKOS_LAMBDA(int idx) { auto i = + interaction_pairs[idx].first; auto j = interaction_pairs[idx].second; + first_neighbor_kernel(i, j); + }); */ - + Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), Cabana::TeamOpTag()); - + Kokkos::fence(); #ifdef CALIPER - CALI_MARK_END("Cabana - calc Force"); + CALI_MARK_END("Cabana - calc Force"); #endif } // Save data for next iteration if we just rebuilt if (rebuild) { - CabanaData new_data(verlet_list, unique_particles, particle_storage.size()); + CabanaData new_data(verlet_list, unique_particles, + particle_storage.size()); cell_structure.set_cabana_data(std::make_unique(new_data)); } @@ -555,12 +560,12 @@ void cabana_short_range( ty += local_torque(tid, i, 1); tz += local_torque(tid, i, 2); } - aosoa.force(i, 0) = fx; - aosoa.force(i, 1) = fy; - aosoa.force(i, 2) = fz; - aosoa.torque(i, 0) = tx; - aosoa.torque(i, 1) = ty; - aosoa.torque(i, 2) = tz; + aosoa.force(i, 0) = fx; + aosoa.force(i, 1) = fy; + aosoa.force(i, 2) = fz; + aosoa.torque(i, 0) = tx; + aosoa.torque(i, 1) = ty; + aosoa.torque(i, 2) = tz; }); Kokkos::fence(); diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 94ecb3c8aac..f5db13788ea 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -53,7 +53,7 @@ inline void set_offset_and_size_indexed_by_cid( int dx[3] = {}; dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / - cell_num[2]); + cell_num[2]); dx[2] = cid % cell_num[2]; bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); @@ -64,12 +64,12 @@ inline void set_offset_and_size_indexed_by_cid( } using ActiveProtocol = std::variant; + LeesEdwards::OscillatoryShear>; inline int set_interacting_pair_cell( int &total_bins, int total_pair_cell, int *cell_num, int *delta_lebc, int le_direction, int le_normal, std::shared_ptr le_protocol, - //ActiveProtocol le_protocol, + // ActiveProtocol le_protocol, Kokkos::View &bin_size, Cabana::LinkedCellList &cell_list, Kokkos::View &interacting_pair_cell) { @@ -94,53 +94,53 @@ inline int set_interacting_pair_cell( bool duplicate_cell = false; // Obtaining 3 dimentional cell index from neighbor cell for (int d = 0; d < 3; ++d) { - dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; - if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) - duplicate_cell = true; + dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; + if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) + duplicate_cell = true; } if (duplicate_cell) - continue; + continue; // Lees-Edwards BC int le_crossing = 0; if (le_protocol != nullptr) { - le_crossing = - ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } + le_crossing = + ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + + cell_num[le_direction]) % + cell_num[le_direction]; + } } // Additional Cell /* if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } - cell_offset = bin_offset(dx[0], dx[1], dx[2]); - cell_size = bin_size(dx[0], dx[1], dx[2]); + if (le_crossing < 0) { + dx[le_direction] = (dx[le_direction] + 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } else if (le_crossing > 0) { + dx[le_direction] = (dx[le_direction] - 1 + + cell_num[le_direction]) % cell_num[le_direction]; + } + cell_offset = bin_offset(dx[0], dx[1], dx[2]); + cell_size = bin_size(dx[0], dx[1], dx[2]); } */ // Interacting pair cell is registered in the list int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); if (cid_i <= cid_j) { - if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - ++pair_cell_id; - } else { - ++empty_pair_number; - } + if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { + interacting_pair_cell(pair_cell_id, 0) = cid_i; + interacting_pair_cell(pair_cell_id, 1) = cid_j; + ++pair_cell_id; + } else { + ++empty_pair_number; + } } } } @@ -149,16 +149,16 @@ inline int set_interacting_pair_cell( using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; -//template + Cabana::VerletLayout2D>; +// template template -ListType create_verlet_list( - double const max_cutoff, int const max_counts, - AoSoA_pack aosoa, - std::vector unique_particles, - VerletCriterion const &verlet_criterion, - Kernel first_neighbor_kernel, - CellStructure &cell_structure) { +ListType create_verlet_list(double const max_cutoff, int const max_counts, + AoSoA_pack aosoa, + std::vector unique_particles, + VerletCriterion const &verlet_criterion, + Kernel first_neighbor_kernel, + CellStructure &cell_structure) { // Creating LinkedCellList and VerletList: // Box Properties auto const &system = ::System::get_system(); @@ -183,7 +183,8 @@ ListType create_verlet_list( int le_normal; int delta_lebc[3] = {0, 0, 0}; auto le_protocol = system.lees_edwards->get_protocol(); - //std::shared_ptr le_protocol = system.lees_edwards->get_protocol(); + // std::shared_ptr le_protocol = + // system.lees_edwards->get_protocol(); if (le_protocol == nullptr) { le_offset = 0.; le_direction = -1; @@ -193,8 +194,8 @@ ListType create_verlet_list( le_direction = box_geo.lees_edwards_bc().shear_direction; le_normal = box_geo.lees_edwards_bc().shear_plane_normal; delta_lebc[le_direction] = - static_cast(std::ceil(le_offset / grid_delta[le_direction])) % - cell_num[le_direction]; + static_cast(std::ceil(le_offset / grid_delta[le_direction])) % + cell_num[le_direction]; } cell_list = Cabana::createLinkedCellList( aosoa.position, grid_delta, grid_min, grid_max); @@ -207,11 +208,10 @@ ListType create_verlet_list( ListType(aosoa.position, 0, aosoa.position.size(), max_counts); // Offset particle id and the number of particle in specific cell - Kokkos::View bin_offset("bin_offset", - total_bins); + Kokkos::View bin_offset("bin_offset", total_bins); Kokkos::View bin_size("bin_size", total_bins); set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, - bin_offset, bin_size); + bin_offset, bin_size); auto const particle_bins = cell_list.getParticleBins(); // Creating Interacting cell @@ -228,57 +228,58 @@ ListType create_verlet_list( le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; + std::as_const(cell_structure).decomposition().box()}; // This kernel used the loop for the pair of interacting cell auto kernel = [&](const int pair_cell_i) { int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); - auto verlet_kernel = [&](Particle *p1, int ii, int id_i, - int cell_offset, int cell_size) { + auto verlet_kernel = [&](Particle *p1, int ii, int id_i, int cell_offset, + int cell_size) { for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - //int ii = cell_list.permutation(i); // debug - // int jj = j; - int jj = cell_list.permutation(j); - int id_j = aosoa.id(jj); - if (aosoa.ghost(ii) or aosoa.ghost(jj)) { - if (((id_i < id_j) and aosoa.ghost(ii)) or - ((id_i > id_j) and aosoa.ghost(jj))) { - continue; - } - } else if (aosoa.ghost(ii) and aosoa.ghost(jj)) { - continue; // reject both ghost - } - auto p2 = unique_particles.at(jj);//cell_structure.get_local_particle(id_j); - //if (p2 == nullptr) - // continue; - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " - << aosoa.ghost(ii) << " " - << aosoa.ghost(jj) << " " - << cid_i << " " - << cid_j << " " - << aosoa.position(ii, 0) << ", " - << aosoa.position(ii, 1) << ", " - << aosoa.position(ii, 2) << " " - << aosoa.position(jj, 0) << ", " - << aosoa.position(jj, 1) << ", " - << aosoa.position(jj, 2) << "\n";// - //std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ - first_neighbor_kernel(ii, jj); - } + // int ii = cell_list.permutation(i); // debug + // int jj = j; + int jj = cell_list.permutation(j); + int id_j = aosoa.id(jj); + if (aosoa.ghost(ii) or aosoa.ghost(jj)) { + if (((id_i < id_j) and aosoa.ghost(ii)) or + ((id_i > id_j) and aosoa.ghost(jj))) { + continue; + } + } else if (aosoa.ghost(ii) and aosoa.ghost(jj)) { + continue; // reject both ghost + } + auto p2 = + unique_particles.at(jj); // cell_structure.get_local_particle(id_j); + // if (p2 == nullptr) + // continue; + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << aosoa.ghost(ii) << " " + << aosoa.ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << aosoa.position(ii, 0) << ", " + << aosoa.position(ii, 1) << ", " + << aosoa.position(ii, 2) << " " + << aosoa.position(jj, 0) << ", " + << aosoa.position(jj, 1) << ", " + << aosoa.position(jj, 2) << "\n";// + //std::cout << "CHECK " + << n << " " + << i << " " + << j << " " + << dx[0] << " " + << dx[1] << " " + << dx[2] << "\n";*/ + first_neighbor_kernel(ii, jj); + } } // j-loop }; @@ -287,28 +288,29 @@ ListType create_verlet_list( for (int i = offset_i; i < offset_i + size_i; ++i) { // int ii = i; - int ii = cell_list.permutation(i); //get previous id + int ii = cell_list.permutation(i); // get previous id int id_i = aosoa.id(ii); - auto p1 = unique_particles.at(ii); //cell_structure.get_local_particle(id_i); - //if (p1 == nullptr) - // continue; + auto p1 = + unique_particles.at(ii); // cell_structure.get_local_particle(id_i); + // if (p1 == nullptr) + // continue; if (cid_i == cid_j) { - verlet_kernel(p1, ii, id_i, i + 1, - size_i + offset_i - i - 1); // j-loop - // verlet_kernel(p1, i, id_i, i + 1, - // size_i + offset_i - i - 1); // j-loop + verlet_kernel(p1, ii, id_i, i + 1, + size_i + offset_i - i - 1); // j-loop + // verlet_kernel(p1, i, id_i, i + 1, + // size_i + offset_i - i - 1); // j-loop } else { - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop - //verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); + verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop + // verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop } } // i-loop }; Kokkos::RangePolicy policy(0, total_pair_cell - - empty_pair_number); + empty_pair_number); Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); From 97a44052628a530166d245e9fa263ce424984ac8 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 25 Jun 2025 13:30:51 +0200 Subject: [PATCH 36/94] Deleted some variables --- src/core/aosoa_pack.hpp | 28 ++++---- src/core/cabana_data.hpp | 13 +--- src/core/custom_verlet_list.hpp | 58 ++++++++++++++-- src/core/verlet_list_loop.hpp | 118 +++++++++++++++++++++----------- 4 files changed, 148 insertions(+), 69 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index 01341a7d473..5ef82d9974e 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -23,28 +23,30 @@ #include -const int vector_length = 1; -using data_types = Cabana::MemberTypes; +const int vector_length = 8; +//using data_types = Cabana::MemberTypes; +using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using AoSoA_type = Cabana::AoSoA; struct AoSoA_pack { AoSoA_type::member_slice_type<0> position; - AoSoA_type::member_slice_type<1> force; - AoSoA_type::member_slice_type<2> torque; - AoSoA_type::member_slice_type<3> charge; - AoSoA_type::member_slice_type<4> id; - AoSoA_type::member_slice_type<5> type; - AoSoA_type::member_slice_type<6> ghost; + //AoSoA_type::member_slice_type<1> force; + //AoSoA_type::member_slice_type<2> torque; + AoSoA_type::member_slice_type<1> charge; + AoSoA_type::member_slice_type<2> id; + AoSoA_type::member_slice_type<3> type; + AoSoA_type::member_slice_type<4> ghost; AoSoA_pack() = default; AoSoA_pack(AoSoA_type &aosoa) - : position(Cabana::slice<0>(aosoa)), force(Cabana::slice<1>(aosoa)), - torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), - id(Cabana::slice<4>(aosoa)), type(Cabana::slice<5>(aosoa)), - ghost(Cabana::slice<6>(aosoa)) {} + : //position(Cabana::slice<0>(aosoa)), force(Cabana::slice<1>(aosoa)), + //torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), + position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), + id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)), + ghost(Cabana::slice<4>(aosoa)) {} }; #endif diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 7b6028e8b7e..13b72a50761 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -34,30 +34,19 @@ using ListType = Cabana::CustomVerletList; class CabanaData { +private: ListType verlet_list; - std::unordered_map id_to_index; - std::vector index_to_id; std::vector unique_particles; int particle_number; public: CabanaData() = default; - CabanaData(ListType verlet_list, std::unordered_map id_to_index, - std::vector index_to_id) - : verlet_list(verlet_list), id_to_index(id_to_index), - index_to_id(index_to_id) {} - CabanaData(ListType verlet_list, std::unordered_map id_to_index) - : verlet_list(verlet_list), id_to_index(id_to_index) {} - CabanaData(ListType verlet_list, int particle_number) - : verlet_list(verlet_list), particle_number(particle_number) {} CabanaData(ListType verlet_list, std::vector unique_particles, int particle_number) : verlet_list(verlet_list), unique_particles(unique_particles), particle_number(particle_number) {} ListType get_verlet_list() const { return verlet_list; } - std::unordered_map get_id_to_index() const { return id_to_index; } - std::vector get_index_to_id() const { return index_to_id; } int get_index() const { return particle_number; } std::vector get_unique_particles() const { return unique_particles; diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 611d83a000c..d60a1ef2bc4 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -38,11 +38,15 @@ class CustomVerletList // Custom constructor template CustomVerletList(PositionSlice x, const std::size_t begin, - const std::size_t end, const std::size_t max_neigh) { - initializeData(x.size(), max_neigh); + const std::size_t end, const std::size_t max_neigh, const std::size_t thread_number) { + initializeData(x.size(), max_neigh, thread_number); } virtual ~CustomVerletList() {}; +private: + Kokkos::View counts_thread; + Kokkos::View neighbors_thread; + public: Kokkos::View counts; Kokkos::View neighbors; @@ -50,18 +54,29 @@ class CustomVerletList // Method to initialize _data without filling neighbors KOKKOS_INLINE_FUNCTION void initializeData(const std::size_t num_particles, - const std::size_t max_neigh) { + const std::size_t max_neigh, + const std::size_t thread_number) { counts = Kokkos::View("num_neighbors", num_particles); neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, max_neigh); + counts_thread = Kokkos::View("num_neighbors", thread_number, num_particles); + neighbors_thread = Kokkos::View( + Kokkos::ViewAllocateWithoutInitializing("neighbors"), thread_number, num_particles, + max_neigh); + Kokkos::parallel_for("initialize counts_thread", num_particles, + [=, this](const int& i) { + for (int tid = 0; tid < thread_number; ++tid) { + counts_thread(tid, i) = 0; + } + }); } // Method to dynamically expand the size of max_neighbors // This function may be vaiolated Kokkos's rule. // Kokkos::View should not be created in Kokkos::parallel. - // However, addNeighbor is used in the Kokkos::parallel and - // this function is called from addNeighbor. + // However, this function is called from addNeighbor used + // in the Kokkos::parallel. KOKKOS_INLINE_FUNCTION void expandMaxNeighbors(const std::size_t new_max_neigh) { // Create a new view with the larger size @@ -92,6 +107,39 @@ class CustomVerletList } neighbors(pid, count) = nid; } + + // Thread safe but non atomic method to add a neighbor + KOKKOS_INLINE_FUNCTION + void addNeighborNonAtomic(const int tid, const int pid, const int nid) { + neighbors_thread(tid, pid, counts_thread(tid, pid)) = nid; + counts_thread(tid, pid) += 1; + if (counts_thread(tid, pid) >= neighbors.extent(1)) { + throw std::runtime_error( + "Number of count in one thread is larger than VerletList size."); + } + } + + // Reduction of counts and neighbor in all threads + void reduction() { + //Kokkos::RangePolicy policy(0, counts.extent(0)); + int thread_number = counts_thread.extent(0); + Kokkos::parallel_for( + "reduction_neighbor", counts.extent(0), [&](const int pid) { + counts(pid) = 0; + for (int tid = 0; tid < thread_number; ++tid) { + std::size_t offset = counts(pid); + counts(pid) += counts_thread(tid, pid); + if (counts(pid) >= neighbors.extent(1)) { + throw std::runtime_error( + "Number of count is larger than VerletList size."); + } + for (int cid = 0; cid < counts_thread(tid, pid); ++cid) { + neighbors(pid, offset + cid) = neighbors_thread(tid, pid, cid); + } + } + }); + Kokkos::fence(); + } }; template diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index f5db13788ea..b9df13a6bf6 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -48,8 +48,14 @@ inline void set_offset_and_size_indexed_by_cid( int &total_bins, int *cell_num, Cabana::LinkedCellList &cell_list, Kokkos::View &bin_offset, - Kokkos::View &bin_size) { - for (int cid = 0; cid < total_bins; ++cid) { + Kokkos::View &bin_size, + Kokkos::View &original_idx) { +#ifdef CALIPER + CALI_CXX_MARK_FUNCTION; +#endif + //for (int cid = 0; cid < total_bins; ++cid) { + Kokkos::parallel_for("set_offset", total_bins, + [&cell_num, &cell_list, &bin_offset, &bin_size](const int cid) { int dx[3] = {}; dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / @@ -60,7 +66,11 @@ inline void set_offset_and_size_indexed_by_cid( // Calculate particle_bins cell_list(cid); - } + }); + Kokkos::parallel_for("set_permutation", original_idx.extent(0), + [&cell_list, &original_idx](const int i) { + original_idx(i) = cell_list.permutation(i); + }); } using ActiveProtocol = std::variant &bin_size, Cabana::LinkedCellList &cell_list, Kokkos::View &interacting_pair_cell) { - +#ifdef CALIPER + CALI_CXX_MARK_FUNCTION; +#endif constexpr int ijkIndexes[27][3] = { {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, @@ -84,13 +96,28 @@ inline int set_interacting_pair_cell( int empty_pair_number = 0; int pair_cell_id = 0; + //Kokkos::View empty_pair_number("empty_pair_number"); + //Kokkos::View pair_cell_id("pair_cell_id"); + //Kokkos::deep_copy(empty_pair_number, 0); + //Kokkos::deep_copy(pair_cell_id, 0); + for (int cid_i = 0; cid_i < total_bins; ++cid_i) { + //Kokkos::parallel_for("set_interacting_pair_cell", total_bins, + // KOKKOS_LAMBDA(const int cid_i) { + //auto thread_id = omp_get_thread_num(); // Obtaining 3 dimentional cell index from cid_i int index[3] = {}; cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); int dx[3]; // From 27 neighbor cell, the list of interacting pair cell is created for (int n = 0; n < 27; ++n) { + + if (le_protocol == nullptr) { + if (index[0] != 0 and ijkIndexes[n][0] == -1) continue; + + if (index[1] != 0 and ijkIndexes[n][1] == -1 and ijkIndexes[n][0] == 0) continue; + } + bool duplicate_cell = false; // Obtaining 3 dimentional cell index from neighbor cell for (int d = 0; d < 3; ++d) { @@ -135,14 +162,23 @@ inline int set_interacting_pair_cell( int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); if (cid_i <= cid_j) { if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { + //std::size_t pcid = Kokkos::atomic_fetch_inc(&pair_cell_id()); + //interacting_pair_cell(pcid, 0) = cid_i; + //interacting_pair_cell(pcid, 1) = cid_j; interacting_pair_cell(pair_cell_id, 0) = cid_i; interacting_pair_cell(pair_cell_id, 1) = cid_j; ++pair_cell_id; + //interacting_pair_cell_thread(thread_id, pair_id_thread(thread_id), 0) = cid_i; + //interacting_pair_cell_thread(thread_id, pair_id_thread(thread_id), 1) = cid_j; + //pair_id_thread(thread_id) += 1; } else { + //Kokkos::atomic_inc(&empty_pair_number()); ++empty_pair_number; + //empty_thread(thread_id) += 1; } } } + //}); } return empty_pair_number; } @@ -150,8 +186,6 @@ inline int set_interacting_pair_cell( using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; -// template template ListType create_verlet_list(double const max_cutoff, int const max_counts, AoSoA_pack aosoa, @@ -183,8 +217,6 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, int le_normal; int delta_lebc[3] = {0, 0, 0}; auto le_protocol = system.lees_edwards->get_protocol(); - // std::shared_ptr le_protocol = - // system.lees_edwards->get_protocol(); if (le_protocol == nullptr) { le_offset = 0.; le_direction = -1; @@ -197,21 +229,31 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, static_cast(std::ceil(le_offset / grid_delta[le_direction])) % cell_num[le_direction]; } +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - CellList"); +#endif cell_list = Cabana::createLinkedCellList( aosoa.position, grid_delta, grid_min, grid_max); +#ifdef CALIPER + CALI_MARK_END("Cabana - CellList"); +#endif int total_bins = cell_list.totalBins(); // Now permute the AoSoA (i.e. reorder the data) using the linked cell // list. // Cabana::permute( cell_list, particle_storage ); + // Number of threads + int num_threads = execution_space().concurrency(); + ListType verlet_list = - ListType(aosoa.position, 0, aosoa.position.size(), max_counts); + ListType(aosoa.position, 0, aosoa.position.size(), max_counts, num_threads); // Offset particle id and the number of particle in specific cell Kokkos::View bin_offset("bin_offset", total_bins); Kokkos::View bin_size("bin_size", total_bins); + Kokkos::View original_idx("original_idx", unique_particles.size()); set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, - bin_offset, bin_size); + bin_offset, bin_size, original_idx); auto const particle_bins = cell_list.getParticleBins(); // Creating Interacting cell @@ -230,39 +272,44 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; + auto aosoa_id = aosoa.id; + auto aosoa_ghost = aosoa.ghost; // This kernel used the loop for the pair of interacting cell - auto kernel = [&](const int pair_cell_i) { + auto kernel = [&interacting_pair_cell, &bin_offset, &bin_size, + &original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, &distance_function, + &verlet_list, &first_neighbor_kernel] (const int pair_cell_i) { + //auto thread_id = omp_get_thread_num(); int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); - auto verlet_kernel = [&](Particle *p1, int ii, int id_i, int cell_offset, - int cell_size) { + auto verlet_kernel = [&original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, + &verlet_criterion, &distance_function, &verlet_list, &first_neighbor_kernel]//, thread_id] + (Particle *p1, int ii, int id_i, int cell_offset, int cell_size) { for (int j = cell_offset; j < cell_offset + cell_size; ++j) { // int ii = cell_list.permutation(i); // debug // int jj = j; - int jj = cell_list.permutation(j); - int id_j = aosoa.id(jj); - if (aosoa.ghost(ii) or aosoa.ghost(jj)) { - if (((id_i < id_j) and aosoa.ghost(ii)) or - ((id_i > id_j) and aosoa.ghost(jj))) { + int jj = original_idx(j); + int id_j = aosoa_id(jj); + if (aosoa_ghost(ii) or aosoa_ghost(jj)) { + if (((id_i < id_j) and aosoa_ghost(ii)) or + ((id_i > id_j) and aosoa_ghost(jj))) { continue; } - } else if (aosoa.ghost(ii) and aosoa.ghost(jj)) { + } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { continue; // reject both ghost } - auto p2 = - unique_particles.at(jj); // cell_structure.get_local_particle(id_j); - // if (p2 == nullptr) - // continue; + auto p2 = unique_particles.at(jj); + //auto p2 = cell_structure.get_local_particle(id_j); if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { verlet_list.addNeighbor(ii, jj); + //verlet_list.addNeighborNonAtomic(thread_id, ii, jj); /*std::cout << "*Cabana* " << i << " " << j << " " << id_i << " " << id_j << " " - << aosoa.ghost(ii) << " " - << aosoa.ghost(jj) << " " + << aosoa_ghost(ii) << " " + << aosoa_ghost(jj) << " " << cid_i << " " << cid_j << " " << aosoa.position(ii, 0) << ", " @@ -270,14 +317,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, << aosoa.position(ii, 2) << " " << aosoa.position(jj, 0) << ", " << aosoa.position(jj, 1) << ", " - << aosoa.position(jj, 2) << "\n";// - //std::cout << "CHECK " - << n << " " - << i << " " - << j << " " - << dx[0] << " " - << dx[1] << " " - << dx[2] << "\n";*/ + << aosoa.position(jj, 2) << "\n";*/ first_neighbor_kernel(ii, jj); } } // j-loop @@ -288,12 +328,10 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, for (int i = offset_i; i < offset_i + size_i; ++i) { // int ii = i; - int ii = cell_list.permutation(i); // get previous id - int id_i = aosoa.id(ii); - auto p1 = - unique_particles.at(ii); // cell_structure.get_local_particle(id_i); - // if (p1 == nullptr) - // continue; + int ii = original_idx(i); // get previous id + int id_i = aosoa_id(ii); + auto p1 = unique_particles.at(ii); + //auto p1 = cell_structure.get_local_particle(id_i); if (cid_i == cid_j) { verlet_kernel(p1, ii, id_i, i + 1, @@ -314,6 +352,8 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); + //verlet_list.reduction(); + return verlet_list; } #endif // SHARED_MEMORY_PARALLELISM From 528e51dac9183e3e50a86e79b3f337f8d7d95593 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 25 Jun 2025 13:31:42 +0200 Subject: [PATCH 37/94] Modified write_particles() --- src/core/short_range_cabana.hpp | 145 ++++++++++++++++---------------- 1 file changed, 72 insertions(+), 73 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 9a52b2ac177..de82139d504 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -44,30 +44,30 @@ inline double wrap(double x, double L) { auto result = x - std::floor(x / L) * L; - if (result >= L) - result -= std::nextafter(L, 0.); + //if (result >= L) + // result -= std::nextafter(L, 0.); return result; } inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, Utils::Vector3d &box_l) { - auto const pos = p.pos(); - aosoa.position(id, 0) = wrap(pos[0], box_l[0]); - aosoa.position(id, 1) = wrap(pos[1], box_l[1]); - aosoa.position(id, 2) = wrap(pos[2], box_l[2]); aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); aosoa.ghost(id) = p.is_ghost(); - aosoa.force(id, 0) = 0.0; - aosoa.force(id, 1) = 0.0; - aosoa.force(id, 2) = 0.0; - aosoa.torque(id, 0) = 0.0; - aosoa.torque(id, 1) = 0.0; - aosoa.torque(id, 2) = 0.0; - assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); - assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); - assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); + auto const pos = p.pos(); + double wpos[3] = {}; + for (int d = 0; d < 3; ++d) { + //aosoa.position(id, d) = + // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + wpos[d] = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + } + for (int d = 0; d < 3; ++d) { + aosoa.position(id, d) = wpos[d]; + } + //assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); + //assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); + //assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); } template @@ -122,7 +122,6 @@ void cabana_short_range( // Number of threads int num_threads = execution_space().concurrency(); - // const int vector_length = 1; #ifdef CALIPER CALI_MARK_END("Cabana - Setup"); #endif @@ -137,6 +136,7 @@ void cabana_short_range( std::unordered_set registered_index{}; // std::vector index_to_id{}; std::vector unique_particles; + //std::vector sequential_particles; int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); @@ -159,6 +159,7 @@ void cabana_short_range( // id_to_index[p.id()] = index; registered_index.insert(p.id()); unique_particles.emplace_back(&p); + //sequential_particles.emplace_back(p); index++; } } @@ -169,6 +170,7 @@ void cabana_short_range( // id_to_index[p.id()] = index; registered_index.insert(p.id()); unique_particles.emplace_back(&p); + //sequential_particles.emplace_back(p); index++; } } @@ -178,6 +180,10 @@ void cabana_short_range( // id_to_index = saved_data.get_id_to_index(); index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); + //sequential_particles.reserve(unique_particles.size()); + //for (Particle * ptr : unique_particles) { + // sequential_particles.emplace_back(*ptr); + //} } int number_of_unique_particles = index; @@ -196,17 +202,37 @@ void cabana_short_range( auto aosoa = AoSoA_pack( particle_storage); // particle properties are defined in aosoa_pack.hpp auto box_l = box_geo.length(); - // int p_id = 0; - // registered_index.clear(); - // Kokkos::View particle_view("particle_pointer", - // unique_particles.size()); - Kokkos::RangePolicy allocation_policy( - 0, unique_particles.size()); - Kokkos::parallel_for("allocation", allocation_policy, [&](int p_id) { + + Kokkos::RangePolicy allocation_policy(0, number_of_unique_particles); + //Kokkos::View device_particles("particles", number_of_unique_particles); + //for (int i = 0; i < number_of_unique_particles; ++i) device_particles(i) = *unique_particles[i]; + Kokkos::parallel_for("allocation", allocation_policy, [&unique_particles, &box_l, &aosoa](int p_id) { + //for (int p_id = 0; p_id < number_of_unique_particles; ++p_id) { + //auto thread_id = omp_get_thread_num(); auto p = *unique_particles[p_id]; - // if (!cell_structure.get_local_particle(p.id())) continue; + //auto p = sequential_particles[p_id]; + //auto p = device_particles(p_id); write_particle(p, p_id, aosoa, box_l); + //} }); + // + /*using policy_type = Kokkos::TeamPolicy; + int soa_length = vector_length; + int num_soa = (number_of_unique_particles + soa_length - 1) / soa_length; + policy_type team_policy(num_soa, Kokkos::AUTO); + Kokkos::parallel_for("allocation", team_policy, + [&](const policy_type::member_type &team_member) { + int soa_idx = team_member.league_rank(); + int start = soa_idx * soa_length; + int end = start + soa_length; + if (end > number_of_unique_particles) end = number_of_unique_particles; + + for (int p_id = start + team_member.team_rank(); p_id < end; + p_id += team_member.team_size()) { + auto p = *unique_particles[p_id]; + write_particle(p, p_id, aosoa, box_l); + } + });*/ Kokkos::fence(); Kokkos::View local_force( @@ -316,9 +342,10 @@ void cabana_short_range( IA_parameters const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); /* - auto p1 = unique_particles.at(i); // -cell->get_local_particle(aosoa.id(i)); auto p2 = unique_particles.at(j);// -cell->get_local_particle(aosoa.id(j)); + auto p1 = unique_particles.at(i); + auto p2 = unique_particles.at(j); + //auto p1 = cell->get_local_particle(aosoa.id(i)); + //auto p2 = cell->get_local_particle(aosoa.id(j)); Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), p2->pos()); auto const dist = d.norm(); @@ -352,10 +379,10 @@ cell->get_local_particle(aosoa.id(j)); auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); #ifdef EXCLUSIONS - auto p1 = - unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); - auto p2 = - unique_particles.at(j); // cell->get_local_particle(aosoa.id(j)); + auto p1 = unique_particles.at(i); + auto p2 = unique_particles.at(j); + //auto p1 = cell->get_local_particle(aosoa.id(i)); + //auto p2 = cell->get_local_particle(aosoa.id(j)); // if (p1 == nullptr or p2 == nullptr) // return; @@ -373,10 +400,10 @@ cell->get_local_particle(aosoa.id(j)); auto const dist2 = dist * dist; #ifndef EXCLUSIONS - auto p1 = - unique_particles.at(i); // cell->get_local_particle(aosoa.id(i)); - auto p2 = - unique_particles.at(j); // cell->get_local_particle(aosoa.id(j)); + auto p1 = unique_particles.at(i); + auto p2 = unique_particles.at(j); + //auto p1 = cell->get_local_particle(aosoa.id(i)); + //auto p2 = cell->get_local_particle(aosoa.id(j)); // if (p1 == nullptr or p2 == nullptr) // return; @@ -442,8 +469,8 @@ cell->get_local_particle(aosoa.id(j)); max_counts = static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); } - if (max_counts < 256) - max_counts = 256; + if (max_counts < 64) + max_counts = 64; if (rebuild) { // Legacy Velert List /*verlet_list = ListType(aosoa.position, 0,aosoa.position.size(), max_counts); @@ -534,8 +561,7 @@ cell->get_local_particle(aosoa.id(j)); // Save data for next iteration if we just rebuilt if (rebuild) { - CabanaData new_data(verlet_list, unique_particles, - particle_storage.size()); + CabanaData new_data(verlet_list, unique_particles, unique_particles.size()); cell_structure.set_cabana_data(std::make_unique(new_data)); } @@ -545,7 +571,7 @@ cell->get_local_particle(aosoa.id(j)); // Force and Torque reduction Kokkos::RangePolicy policy(0, particle_storage.size()); Kokkos::parallel_for( - "reduction", policy, KOKKOS_LAMBDA(const int i) { + "reduction", policy, [&local_force, &local_torque, &unique_particles, num_threads](const int i) { double fx = 0.; double fy = 0.; double fz = 0.; @@ -560,12 +586,12 @@ cell->get_local_particle(aosoa.id(j)); ty += local_torque(tid, i, 1); tz += local_torque(tid, i, 2); } - aosoa.force(i, 0) = fx; - aosoa.force(i, 1) = fy; - aosoa.force(i, 2) = fz; - aosoa.torque(i, 0) = tx; - aosoa.torque(i, 1) = ty; - aosoa.torque(i, 2) = tz; + auto &p = unique_particles[i]; + //auto p = cell_structure.get_local_particle(aosoa.id(i)); + p->force() += Utils::Vector3d{fx, fy, fz}; +#ifdef ROTATION + p->torque() += Utils::Vector3d{tx, ty, tz}; +#endif }); Kokkos::fence(); @@ -600,33 +626,6 @@ cell->get_local_particle(aosoa.id(j)); #ifdef CALIPER CALI_MARK_END("Cabana - Collision Detection"); #endif - - // =================================================== - // Add forces to particles - // =================================================== -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Particle Forces"); -#endif - for (auto id = 0; id < particle_storage.size(); ++id) { - auto p = cell_structure.get_local_particle(aosoa.id(id)); - if (p == nullptr) { - return; - } - Utils::Vector3d f_vec{aosoa.force(id, 0), aosoa.force(id, 1), - aosoa.force(id, 2)}; - Utils::Vector3d torque_vec{aosoa.torque(id, 0), aosoa.torque(id, 1), - aosoa.torque(id, 2)}; - -#ifdef ROTATION - ParticleForce f(f_vec, torque_vec); -#else - ParticleForce f(f_vec); -#endif - p->force_and_torque() += f; - } -#ifdef CALIPER - CALI_MARK_END("Cabana - Particle Forces"); -#endif } } From f688de41f5c869ed8879e0a8e95e1c0a838060bb Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 25 Jun 2025 13:33:01 +0200 Subject: [PATCH 38/94] Formatting --- src/core/aosoa_pack.hpp | 13 +-- src/core/custom_verlet_list.hpp | 42 ++++---- src/core/short_range_cabana.hpp | 114 ++++++++++---------- src/core/verlet_list_loop.hpp | 180 +++++++++++++++++--------------- 4 files changed, 184 insertions(+), 165 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index 5ef82d9974e..d0234e0d10f 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -24,8 +24,9 @@ #include const int vector_length = 8; -//using data_types = Cabana::MemberTypes; +// using data_types = Cabana::MemberTypes; using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -33,8 +34,8 @@ using AoSoA_type = Cabana::AoSoA; struct AoSoA_pack { AoSoA_type::member_slice_type<0> position; - //AoSoA_type::member_slice_type<1> force; - //AoSoA_type::member_slice_type<2> torque; + // AoSoA_type::member_slice_type<1> force; + // AoSoA_type::member_slice_type<2> torque; AoSoA_type::member_slice_type<1> charge; AoSoA_type::member_slice_type<2> id; AoSoA_type::member_slice_type<3> type; @@ -43,8 +44,8 @@ struct AoSoA_pack { AoSoA_pack() = default; AoSoA_pack(AoSoA_type &aosoa) - : //position(Cabana::slice<0>(aosoa)), force(Cabana::slice<1>(aosoa)), - //torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), + : // position(Cabana::slice<0>(aosoa)), force(Cabana::slice<1>(aosoa)), + // torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)), ghost(Cabana::slice<4>(aosoa)) {} diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index d60a1ef2bc4..85c3487fd8f 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -38,7 +38,8 @@ class CustomVerletList // Custom constructor template CustomVerletList(PositionSlice x, const std::size_t begin, - const std::size_t end, const std::size_t max_neigh, const std::size_t thread_number) { + const std::size_t end, const std::size_t max_neigh, + const std::size_t thread_number) { initializeData(x.size(), max_neigh, thread_number); } virtual ~CustomVerletList() {}; @@ -60,16 +61,17 @@ class CustomVerletList neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, max_neigh); - counts_thread = Kokkos::View("num_neighbors", thread_number, num_particles); + counts_thread = Kokkos::View( + "num_neighbors", thread_number, num_particles); neighbors_thread = Kokkos::View( - Kokkos::ViewAllocateWithoutInitializing("neighbors"), thread_number, num_particles, - max_neigh); + Kokkos::ViewAllocateWithoutInitializing("neighbors"), thread_number, + num_particles, max_neigh); Kokkos::parallel_for("initialize counts_thread", num_particles, - [=, this](const int& i) { - for (int tid = 0; tid < thread_number; ++tid) { - counts_thread(tid, i) = 0; - } - }); + [=, this](const int &i) { + for (int tid = 0; tid < thread_number; ++tid) { + counts_thread(tid, i) = 0; + } + }); } // Method to dynamically expand the size of max_neighbors @@ -108,7 +110,7 @@ class CustomVerletList neighbors(pid, count) = nid; } - // Thread safe but non atomic method to add a neighbor + // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION void addNeighborNonAtomic(const int tid, const int pid, const int nid) { neighbors_thread(tid, pid, counts_thread(tid, pid)) = nid; @@ -121,21 +123,21 @@ class CustomVerletList // Reduction of counts and neighbor in all threads void reduction() { - //Kokkos::RangePolicy policy(0, counts.extent(0)); + // Kokkos::RangePolicy policy(0, counts.extent(0)); int thread_number = counts_thread.extent(0); Kokkos::parallel_for( "reduction_neighbor", counts.extent(0), [&](const int pid) { - counts(pid) = 0; + counts(pid) = 0; for (int tid = 0; tid < thread_number; ++tid) { - std::size_t offset = counts(pid); + std::size_t offset = counts(pid); counts(pid) += counts_thread(tid, pid); - if (counts(pid) >= neighbors.extent(1)) { - throw std::runtime_error( - "Number of count is larger than VerletList size."); - } - for (int cid = 0; cid < counts_thread(tid, pid); ++cid) { - neighbors(pid, offset + cid) = neighbors_thread(tid, pid, cid); - } + if (counts(pid) >= neighbors.extent(1)) { + throw std::runtime_error( + "Number of count is larger than VerletList size."); + } + for (int cid = 0; cid < counts_thread(tid, pid); ++cid) { + neighbors(pid, offset + cid) = neighbors_thread(tid, pid, cid); + } } }); Kokkos::fence(); diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index de82139d504..d3aa9c9972a 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -44,8 +44,8 @@ inline double wrap(double x, double L) { auto result = x - std::floor(x / L) * L; - //if (result >= L) - // result -= std::nextafter(L, 0.); + // if (result >= L) + // result -= std::nextafter(L, 0.); return result; } @@ -58,16 +58,16 @@ inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, auto const pos = p.pos(); double wpos[3] = {}; for (int d = 0; d < 3; ++d) { - //aosoa.position(id, d) = - // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + // aosoa.position(id, d) = + // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; wpos[d] = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; } for (int d = 0; d < 3; ++d) { aosoa.position(id, d) = wpos[d]; } - //assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); - //assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); - //assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); + // assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); + // assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); + // assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); } template @@ -136,7 +136,7 @@ void cabana_short_range( std::unordered_set registered_index{}; // std::vector index_to_id{}; std::vector unique_particles; - //std::vector sequential_particles; + // std::vector sequential_particles; int index = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); @@ -159,7 +159,7 @@ void cabana_short_range( // id_to_index[p.id()] = index; registered_index.insert(p.id()); unique_particles.emplace_back(&p); - //sequential_particles.emplace_back(p); + // sequential_particles.emplace_back(p); index++; } } @@ -170,7 +170,7 @@ void cabana_short_range( // id_to_index[p.id()] = index; registered_index.insert(p.id()); unique_particles.emplace_back(&p); - //sequential_particles.emplace_back(p); + // sequential_particles.emplace_back(p); index++; } } @@ -180,10 +180,10 @@ void cabana_short_range( // id_to_index = saved_data.get_id_to_index(); index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); - //sequential_particles.reserve(unique_particles.size()); - //for (Particle * ptr : unique_particles) { + // sequential_particles.reserve(unique_particles.size()); + // for (Particle * ptr : unique_particles) { // sequential_particles.emplace_back(*ptr); - //} + // } } int number_of_unique_particles = index; @@ -203,32 +203,37 @@ void cabana_short_range( particle_storage); // particle properties are defined in aosoa_pack.hpp auto box_l = box_geo.length(); - Kokkos::RangePolicy allocation_policy(0, number_of_unique_particles); - //Kokkos::View device_particles("particles", number_of_unique_particles); - //for (int i = 0; i < number_of_unique_particles; ++i) device_particles(i) = *unique_particles[i]; - Kokkos::parallel_for("allocation", allocation_policy, [&unique_particles, &box_l, &aosoa](int p_id) { - //for (int p_id = 0; p_id < number_of_unique_particles; ++p_id) { - //auto thread_id = omp_get_thread_num(); - auto p = *unique_particles[p_id]; - //auto p = sequential_particles[p_id]; - //auto p = device_particles(p_id); - write_particle(p, p_id, aosoa, box_l); - //} - }); + Kokkos::RangePolicy allocation_policy( + 0, number_of_unique_particles); + // Kokkos::View device_particles("particles", + // number_of_unique_particles); for (int i = 0; i < + // number_of_unique_particles; ++i) device_particles(i) = + // *unique_particles[i]; + Kokkos::parallel_for("allocation", allocation_policy, + [&unique_particles, &box_l, &aosoa](int p_id) { + // for (int p_id = 0; p_id < + // number_of_unique_particles; ++p_id) { auto + // thread_id = omp_get_thread_num(); + auto p = *unique_particles[p_id]; + // auto p = sequential_particles[p_id]; + // auto p = device_particles(p_id); + write_particle(p, p_id, aosoa, box_l); + //} + }); // /*using policy_type = Kokkos::TeamPolicy; int soa_length = vector_length; int num_soa = (number_of_unique_particles + soa_length - 1) / soa_length; policy_type team_policy(num_soa, Kokkos::AUTO); Kokkos::parallel_for("allocation", team_policy, - [&](const policy_type::member_type &team_member) { + [&](const policy_type::member_type &team_member) { int soa_idx = team_member.league_rank(); int start = soa_idx * soa_length; int end = start + soa_length; if (end > number_of_unique_particles) end = number_of_unique_particles; for (int p_id = start + team_member.team_rank(); p_id < end; - p_id += team_member.team_size()) { + p_id += team_member.team_size()) { auto p = *unique_particles[p_id]; write_particle(p, p_id, aosoa, box_l); } @@ -381,8 +386,8 @@ void cabana_short_range( #ifdef EXCLUSIONS auto p1 = unique_particles.at(i); auto p2 = unique_particles.at(j); - //auto p1 = cell->get_local_particle(aosoa.id(i)); - //auto p2 = cell->get_local_particle(aosoa.id(j)); + // auto p1 = cell->get_local_particle(aosoa.id(i)); + // auto p2 = cell->get_local_particle(aosoa.id(j)); // if (p1 == nullptr or p2 == nullptr) // return; @@ -402,8 +407,8 @@ void cabana_short_range( #ifndef EXCLUSIONS auto p1 = unique_particles.at(i); auto p2 = unique_particles.at(j); - //auto p1 = cell->get_local_particle(aosoa.id(i)); - //auto p2 = cell->get_local_particle(aosoa.id(j)); + // auto p1 = cell->get_local_particle(aosoa.id(i)); + // auto p2 = cell->get_local_particle(aosoa.id(j)); // if (p1 == nullptr or p2 == nullptr) // return; @@ -561,7 +566,8 @@ void cabana_short_range( // Save data for next iteration if we just rebuilt if (rebuild) { - CabanaData new_data(verlet_list, unique_particles, unique_particles.size()); + CabanaData new_data(verlet_list, unique_particles, + unique_particles.size()); cell_structure.set_cabana_data(std::make_unique(new_data)); } @@ -570,29 +576,31 @@ void cabana_short_range( #endif // Force and Torque reduction Kokkos::RangePolicy policy(0, particle_storage.size()); - Kokkos::parallel_for( - "reduction", policy, [&local_force, &local_torque, &unique_particles, num_threads](const int i) { - double fx = 0.; - double fy = 0.; - double fz = 0.; - double tx = 0.; - double ty = 0.; - double tz = 0.; - for (int tid = 0; tid < num_threads; ++tid) { - fx += local_force(tid, i, 0); - fy += local_force(tid, i, 1); - fz += local_force(tid, i, 2); - tx += local_torque(tid, i, 0); - ty += local_torque(tid, i, 1); - tz += local_torque(tid, i, 2); - } - auto &p = unique_particles[i]; - //auto p = cell_structure.get_local_particle(aosoa.id(i)); - p->force() += Utils::Vector3d{fx, fy, fz}; + Kokkos::parallel_for("reduction", policy, + [&local_force, &local_torque, &unique_particles, + num_threads](const int i) { + double fx = 0.; + double fy = 0.; + double fz = 0.; + double tx = 0.; + double ty = 0.; + double tz = 0.; + for (int tid = 0; tid < num_threads; ++tid) { + fx += local_force(tid, i, 0); + fy += local_force(tid, i, 1); + fz += local_force(tid, i, 2); + tx += local_torque(tid, i, 0); + ty += local_torque(tid, i, 1); + tz += local_torque(tid, i, 2); + } + auto &p = unique_particles[i]; + // auto p = + // cell_structure.get_local_particle(aosoa.id(i)); + p->force() += Utils::Vector3d{fx, fy, fz}; #ifdef ROTATION - p->torque() += Utils::Vector3d{tx, ty, tz}; + p->torque() += Utils::Vector3d{tx, ty, tz}; #endif - }); + }); Kokkos::fence(); #ifdef NPT diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index b9df13a6bf6..fd248b7e783 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -53,24 +53,25 @@ inline void set_offset_and_size_indexed_by_cid( #ifdef CALIPER CALI_CXX_MARK_FUNCTION; #endif - //for (int cid = 0; cid < total_bins; ++cid) { - Kokkos::parallel_for("set_offset", total_bins, - [&cell_num, &cell_list, &bin_offset, &bin_size](const int cid) { - int dx[3] = {}; - dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); - dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / - cell_num[2]); - dx[2] = cid % cell_num[2]; - bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); - bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); + // for (int cid = 0; cid < total_bins; ++cid) { + Kokkos::parallel_for( + "set_offset", total_bins, + [&cell_num, &cell_list, &bin_offset, &bin_size](const int cid) { + int dx[3] = {}; + dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); + dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / + cell_num[2]); + dx[2] = cid % cell_num[2]; + bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); + bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); - // Calculate particle_bins - cell_list(cid); - }); + // Calculate particle_bins + cell_list(cid); + }); Kokkos::parallel_for("set_permutation", original_idx.extent(0), - [&cell_list, &original_idx](const int i) { - original_idx(i) = cell_list.permutation(i); - }); + [&cell_list, &original_idx](const int i) { + original_idx(i) = cell_list.permutation(i); + }); } using ActiveProtocol = std::variant empty_pair_number("empty_pair_number"); - //Kokkos::View pair_cell_id("pair_cell_id"); - //Kokkos::deep_copy(empty_pair_number, 0); - //Kokkos::deep_copy(pair_cell_id, 0); + // Kokkos::View empty_pair_number("empty_pair_number"); + // Kokkos::View pair_cell_id("pair_cell_id"); + // Kokkos::deep_copy(empty_pair_number, 0); + // Kokkos::deep_copy(pair_cell_id, 0); for (int cid_i = 0; cid_i < total_bins; ++cid_i) { - //Kokkos::parallel_for("set_interacting_pair_cell", total_bins, - // KOKKOS_LAMBDA(const int cid_i) { - //auto thread_id = omp_get_thread_num(); - // Obtaining 3 dimentional cell index from cid_i + // Kokkos::parallel_for("set_interacting_pair_cell", total_bins, + // KOKKOS_LAMBDA(const int cid_i) { + // auto thread_id = omp_get_thread_num(); + // Obtaining 3 dimentional cell index from cid_i int index[3] = {}; cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); int dx[3]; @@ -113,9 +114,11 @@ inline int set_interacting_pair_cell( for (int n = 0; n < 27; ++n) { if (le_protocol == nullptr) { - if (index[0] != 0 and ijkIndexes[n][0] == -1) continue; + if (index[0] != 0 and ijkIndexes[n][0] == -1) + continue; - if (index[1] != 0 and ijkIndexes[n][1] == -1 and ijkIndexes[n][0] == 0) continue; + if (index[1] != 0 and ijkIndexes[n][1] == -1 and ijkIndexes[n][0] == 0) + continue; } bool duplicate_cell = false; @@ -162,23 +165,24 @@ inline int set_interacting_pair_cell( int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); if (cid_i <= cid_j) { if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { - //std::size_t pcid = Kokkos::atomic_fetch_inc(&pair_cell_id()); - //interacting_pair_cell(pcid, 0) = cid_i; - //interacting_pair_cell(pcid, 1) = cid_j; + // std::size_t pcid = Kokkos::atomic_fetch_inc(&pair_cell_id()); + // interacting_pair_cell(pcid, 0) = cid_i; + // interacting_pair_cell(pcid, 1) = cid_j; interacting_pair_cell(pair_cell_id, 0) = cid_i; interacting_pair_cell(pair_cell_id, 1) = cid_j; ++pair_cell_id; - //interacting_pair_cell_thread(thread_id, pair_id_thread(thread_id), 0) = cid_i; - //interacting_pair_cell_thread(thread_id, pair_id_thread(thread_id), 1) = cid_j; - //pair_id_thread(thread_id) += 1; + // interacting_pair_cell_thread(thread_id, pair_id_thread(thread_id), + // 0) = cid_i; interacting_pair_cell_thread(thread_id, + // pair_id_thread(thread_id), 1) = cid_j; pair_id_thread(thread_id) += + // 1; } else { - //Kokkos::atomic_inc(&empty_pair_number()); + // Kokkos::atomic_inc(&empty_pair_number()); ++empty_pair_number; - //empty_thread(thread_id) += 1; + // empty_thread(thread_id) += 1; } } } - //}); + //}); } return empty_pair_number; } @@ -230,12 +234,12 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, cell_num[le_direction]; } #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - CellList"); + CALI_MARK_BEGIN("Cabana - CellList"); #endif cell_list = Cabana::createLinkedCellList( aosoa.position, grid_delta, grid_min, grid_max); #ifdef CALIPER - CALI_MARK_END("Cabana - CellList"); + CALI_MARK_END("Cabana - CellList"); #endif int total_bins = cell_list.totalBins(); // Now permute the AoSoA (i.e. reorder the data) using the linked cell @@ -245,13 +249,14 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // Number of threads int num_threads = execution_space().concurrency(); - ListType verlet_list = - ListType(aosoa.position, 0, aosoa.position.size(), max_counts, num_threads); + ListType verlet_list = ListType(aosoa.position, 0, aosoa.position.size(), + max_counts, num_threads); // Offset particle id and the number of particle in specific cell Kokkos::View bin_offset("bin_offset", total_bins); Kokkos::View bin_size("bin_size", total_bins); - Kokkos::View original_idx("original_idx", unique_particles.size()); + Kokkos::View original_idx( + "original_idx", unique_particles.size()); set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, bin_offset, bin_size, original_idx); auto const particle_bins = cell_list.getParticleBins(); @@ -275,53 +280,56 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto aosoa_id = aosoa.id; auto aosoa_ghost = aosoa.ghost; // This kernel used the loop for the pair of interacting cell - auto kernel = [&interacting_pair_cell, &bin_offset, &bin_size, - &original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, &distance_function, - &verlet_list, &first_neighbor_kernel] (const int pair_cell_i) { - //auto thread_id = omp_get_thread_num(); + auto kernel = [&interacting_pair_cell, &bin_offset, &bin_size, &original_idx, + &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, + &distance_function, &verlet_list, + &first_neighbor_kernel](const int pair_cell_i) { + // auto thread_id = omp_get_thread_num(); int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); - auto verlet_kernel = [&original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, - &verlet_criterion, &distance_function, &verlet_list, &first_neighbor_kernel]//, thread_id] - (Particle *p1, int ii, int id_i, int cell_offset, int cell_size) { - for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - // int ii = cell_list.permutation(i); // debug - // int jj = j; - int jj = original_idx(j); - int id_j = aosoa_id(jj); - if (aosoa_ghost(ii) or aosoa_ghost(jj)) { - if (((id_i < id_j) and aosoa_ghost(ii)) or - ((id_i > id_j) and aosoa_ghost(jj))) { - continue; - } - } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { - continue; // reject both ghost - } - auto p2 = unique_particles.at(jj); - //auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); - //verlet_list.addNeighborNonAtomic(thread_id, ii, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " - << aosoa_ghost(ii) << " " - << aosoa_ghost(jj) << " " - << cid_i << " " - << cid_j << " " - << aosoa.position(ii, 0) << ", " - << aosoa.position(ii, 1) << ", " - << aosoa.position(ii, 2) << " " - << aosoa.position(jj, 0) << ", " - << aosoa.position(jj, 1) << ", " - << aosoa.position(jj, 2) << "\n";*/ - first_neighbor_kernel(ii, jj); - } - } // j-loop - }; + auto verlet_kernel = [&original_idx, &aosoa_id, &aosoa_ghost, + &unique_particles, &verlet_criterion, + &distance_function, &verlet_list, + &first_neighbor_kernel] //, thread_id] + (Particle * p1, int ii, int id_i, int cell_offset, int cell_size) { + for (int j = cell_offset; j < cell_offset + cell_size; ++j) { + // int ii = cell_list.permutation(i); // debug + // int jj = j; + int jj = original_idx(j); + int id_j = aosoa_id(jj); + if (aosoa_ghost(ii) or aosoa_ghost(jj)) { + if (((id_i < id_j) and aosoa_ghost(ii)) or + ((id_i > id_j) and aosoa_ghost(jj))) { + continue; + } + } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { + continue; // reject both ghost + } + auto p2 = unique_particles.at(jj); + // auto p2 = cell_structure.get_local_particle(id_j); + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(ii, jj); + // verlet_list.addNeighborNonAtomic(thread_id, ii, jj); + /*std::cout << "*Cabana* " + << i << " " + << j << " " + << id_i << " " + << id_j << " " + << aosoa_ghost(ii) << " " + << aosoa_ghost(jj) << " " + << cid_i << " " + << cid_j << " " + << aosoa.position(ii, 0) << ", " + << aosoa.position(ii, 1) << ", " + << aosoa.position(ii, 2) << " " + << aosoa.position(jj, 0) << ", " + << aosoa.position(jj, 1) << ", " + << aosoa.position(jj, 2) << "\n";*/ + first_neighbor_kernel(ii, jj); + } + } // j-loop + }; int offset_i = bin_offset(cid_i); int size_i = bin_size(cid_i); @@ -331,7 +339,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, int ii = original_idx(i); // get previous id int id_i = aosoa_id(ii); auto p1 = unique_particles.at(ii); - //auto p1 = cell_structure.get_local_particle(id_i); + // auto p1 = cell_structure.get_local_particle(id_i); if (cid_i == cid_j) { verlet_kernel(p1, ii, id_i, i + 1, @@ -352,7 +360,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, Kokkos::parallel_for("calc_by_cell_list", policy, kernel); Kokkos::fence(); - //verlet_list.reduction(); + // verlet_list.reduction(); return verlet_list; } From a79806618db8f206857600294a2fbd9ecd53b9f5 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Sat, 28 Jun 2025 15:18:17 +0200 Subject: [PATCH 39/94] Implemented optimizing max_counts for VerletList --- src/core/aosoa_pack.hpp | 5 +- src/core/cabana_data.hpp | 9 +- src/core/cell_system/CellStructure.hpp | 4 + src/core/integrate.cpp | 8 + src/core/short_range_cabana.hpp | 218 ++++++++++++------------- src/core/verlet_list_loop.hpp | 27 +-- testsuite/python/exclusions.py | 3 +- 7 files changed, 143 insertions(+), 131 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index d0234e0d10f..d745bb82492 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -23,10 +23,7 @@ #include -const int vector_length = 8; -// using data_types = Cabana::MemberTypes; +const int vector_length = 1; using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 13b72a50761..cc82c58e427 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -37,17 +37,18 @@ class CabanaData { private: ListType verlet_list; std::vector unique_particles; - int particle_number; + int max_id; public: CabanaData() = default; CabanaData(ListType verlet_list, std::vector unique_particles, - int particle_number) + int max_id) : verlet_list(verlet_list), unique_particles(unique_particles), - particle_number(particle_number) {} + max_id(max_id) {} ListType get_verlet_list() const { return verlet_list; } - int get_index() const { return particle_number; } + int get_index() const { return unique_particles.size(); } + int get_max_id() const { return max_id; } std::vector get_unique_particles() const { return unique_particles; } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 147e949d702..7db44629d0d 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -663,6 +663,7 @@ struct CellStructure : public System::Leaf { #ifdef SHARED_MEMORY_PARALLELISM private: std::unique_ptr m_cabana_data; + bool steepest_descent_flag = true; public: void set_cabana_data(std::unique_ptr data); @@ -676,6 +677,9 @@ struct CellStructure : public System::Leaf { return m_rebuild_cabana_verlet_list; } + void set_steepest_descent_flag(bool flag) { steepest_descent_flag = flag; } + bool get_steepest_descent_flag() { return steepest_descent_flag; } + template void cabana_link_cell(Kernel kernel) { auto const local_cells_span = decomposition().local_cells(); auto const first = boost::make_indirect_iterator(local_cells_span.begin()); diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index 19c38cc9d44..e3c372344f3 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -525,7 +525,15 @@ int System::System::integrate(int n_steps, int reuse_forces) { if (propagation.integ_switch != INTEG_METHOD_STEEPEST_DESCENT) { lb_active = lb.is_solver_set(); ek_active = ek.is_ready_for_propagation(); +#ifdef SHARED_MEMORY_PARALLELISM + cell_structure->set_steepest_descent_flag(false); +#endif + } +#ifdef SHARED_MEMORY_PARALLELISM + else { + cell_structure->set_steepest_descent_flag(true); } +#endif auto const calc_md_steps_per_tau = [this](double tau) { return static_cast(std::round(tau / time_step)); }; diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index d3aa9c9972a..25c58720d27 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -132,16 +132,17 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Index map"); #endif - std::unordered_map id_to_index{}; // For DEBUG std::unordered_set registered_index{}; - // std::vector index_to_id{}; std::vector unique_particles; - // std::vector sequential_particles; + //std::vector sequential_particles; int index = 0; + int max_id = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); - // std::cout << "For CABANA rebuild " << rebuild << " " << - // Kokkos::OpenMP::concurrency() << std::endl; + //if (rank == 0) { + // std::cout << "For CABANA rebuild " << rebuild + // << " " << Kokkos::OpenMP::concurrency() << std::endl; + //} CabanaData saved_data; @@ -156,10 +157,10 @@ void cabana_short_range( for (auto &p : particles) { if (cell_structure.get_local_particle(p.id())) { - // id_to_index[p.id()] = index; + if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); - // sequential_particles.emplace_back(p); + //sequential_particles.emplace_back(p); index++; } } @@ -167,23 +168,23 @@ void cabana_short_range( for (auto &p : ghost_particles) { if (not registered_index.contains(p.id())) { if (cell_structure.get_local_particle(p.id())) { - // id_to_index[p.id()] = index; + if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); - // sequential_particles.emplace_back(p); + //sequential_particles.emplace_back(p); index++; } } } } else { // If we do not rebuild we can use the saved map - // id_to_index = saved_data.get_id_to_index(); index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); - // sequential_particles.reserve(unique_particles.size()); - // for (Particle * ptr : unique_particles) { + max_id = saved_data.get_max_id(); + //sequential_particles.reserve(unique_particles.size()); + //for (Particle * ptr : unique_particles) { // sequential_particles.emplace_back(*ptr); - // } + //} } int number_of_unique_particles = index; @@ -199,45 +200,24 @@ void cabana_short_range( #endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); - auto aosoa = AoSoA_pack( - particle_storage); // particle properties are defined in aosoa_pack.hpp + // particle properties are defined in aosoa_pack.hpp + auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); Kokkos::RangePolicy allocation_policy( 0, number_of_unique_particles); - // Kokkos::View device_particles("particles", - // number_of_unique_particles); for (int i = 0; i < - // number_of_unique_particles; ++i) device_particles(i) = - // *unique_particles[i]; - Kokkos::parallel_for("allocation", allocation_policy, - [&unique_particles, &box_l, &aosoa](int p_id) { - // for (int p_id = 0; p_id < - // number_of_unique_particles; ++p_id) { auto - // thread_id = omp_get_thread_num(); - auto p = *unique_particles[p_id]; - // auto p = sequential_particles[p_id]; - // auto p = device_particles(p_id); - write_particle(p, p_id, aosoa, box_l); - //} - }); - // - /*using policy_type = Kokkos::TeamPolicy; - int soa_length = vector_length; - int num_soa = (number_of_unique_particles + soa_length - 1) / soa_length; - policy_type team_policy(num_soa, Kokkos::AUTO); - Kokkos::parallel_for("allocation", team_policy, - [&](const policy_type::member_type &team_member) { - int soa_idx = team_member.league_rank(); - int start = soa_idx * soa_length; - int end = start + soa_length; - if (end > number_of_unique_particles) end = number_of_unique_particles; - - for (int p_id = start + team_member.team_rank(); p_id < end; - p_id += team_member.team_size()) { - auto p = *unique_particles[p_id]; - write_particle(p, p_id, aosoa, box_l); - } - });*/ + Kokkos::View id_to_index("id_to_index", max_id + 1); + //Kokkos::parallel_for("allocation", allocation_policy, + // [&unique_particles, &box_l, &aosoa, &id_to_index](int p_id) { + for (int p_id = 0; p_id < number_of_unique_particles; ++p_id) { + //auto thread_id = omp_get_thread_num(); + auto p = *unique_particles[p_id]; + //auto p = sequential_particles[p_id]; + // auto p = device_particles(p_id); + write_particle(p, p_id, aosoa, box_l); + id_to_index(p.id()) = p_id; + } + //}); Kokkos::fence(); Kokkos::View local_force( @@ -262,11 +242,14 @@ void cabana_short_range( [[maybe_unused]] const BondedInteractionsMap &bonded_ias; const InteractionsNonBonded &nonbonded_ias; const BoxGeometry &box_geo; - // std::vector &index_to_id; + AoSoA_pack aosoa; Kokkos::View local_force; +#ifdef ROTATION Kokkos::View local_torque; +#endif +#ifdef NPT Kokkos::View local_virial; - AoSoA_pack aosoa; +#endif #ifdef COLLISION_DETECTION // std::shared_ptr // collision_detection; @@ -294,10 +277,14 @@ void cabana_short_range( [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, const BoxGeometry &box_geo_, - // std::vector &index_to_id_, + AoSoA_pack &aosoa_, Kokkos::View local_force_, +#ifdef ROTATION Kokkos::View local_torque_, - Kokkos::View local_virial_, AoSoA_pack &aosoa_, +#endif +#ifdef NPT + Kokkos::View local_virial_, +#endif #ifdef COLLISION_DETECTION // std::shared_ptr // collision_detection_, @@ -320,9 +307,14 @@ void cabana_short_range( #endif bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), box_geo(box_geo_), - // index_to_id(index_to_id_), - local_force(local_force_), local_torque(local_torque_), - local_virial(local_virial_), aosoa(aosoa_), + aosoa(aosoa_), + local_force(local_force_), +#ifdef ROTATION + local_torque(local_torque_), +#endif +#ifdef NPT + local_virial(local_virial_), +#endif #ifdef COLLISION_DETECTION collision_detection(collision_detection_), #endif @@ -346,28 +338,6 @@ void cabana_short_range( IA_parameters const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); - /* - auto p1 = unique_particles.at(i); - auto p2 = unique_particles.at(j); - //auto p1 = cell->get_local_particle(aosoa.id(i)); - //auto p2 = cell->get_local_particle(aosoa.id(j)); - - Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), p2->pos()); - auto const dist = d.norm(); - - auto const q1q2 =aosoa.charge(i) *aosoa.charge(j); - - auto const dist2 = dist * dist; -#ifdef NPT - auto [pf, virial] -#else - auto pf -#endif - = add_non_bonded_pair_force( - const_cast(*p1), const_cast(*p2), d, dist, - dist2, q1q2, ia_params, thermostat, box_geo, bonded_ias, - coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); - */ ParticleForce pf{}; #ifdef NPT @@ -465,35 +435,49 @@ void cabana_short_range( ListType verlet_list; // Rebuild verlet list if needed - // auto const &system = ::System::get_system(); + bool at_steepest_descent = cell_structure.get_steepest_descent_flag(); int max_counts; double max_cutoff = pair_cutoff; // system.get_interaction_range(); if (std::isinf(max_cutoff)) { max_counts = number_of_unique_particles; } else { + int max_prefactor; + if (at_steepest_descent) { + max_prefactor = 8; + } else { + max_prefactor = 6; + } max_counts = - static_cast(27 * max_cutoff * max_cutoff * max_cutoff / 3); + static_cast(std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); + } + int threshold_num = 8; +#ifdef COLLISION_DETECTION + threshold_num = 64; +#endif + if (max_counts < threshold_num) { + max_counts = std::min(threshold_num, number_of_unique_particles); } - if (max_counts < 64) - max_counts = 64; + //std::cout << "max_counts:" << max_counts << " " << max_cutoff << std::endl; if (rebuild) { // Legacy Velert List - /*verlet_list = - ListType(aosoa.position, 0,aosoa.position.size(), max_counts); + if (0) { + verlet_list = + ListType(aosoa.position, 0, aosoa.position.size(), max_counts, num_threads); auto kernel = [&](Particle const &p1, Particle const &p2) { - verlet_list.addNeighbor(id_to_index.at(p1.id()), - id_to_index.at(p2.id())); - //std::cout << "WITHSMP " - //<< id_to_index.at(p1.id()) << " " - //<< id_to_index.at(p2.id()) << " " - //<< p1.is_ghost() << " " - //<< p2.is_ghost() << " " - //<< p1.id() << " " - //<< p2.id() << std::endl; + verlet_list.addNeighbor(id_to_index(p1.id()), + id_to_index(p2.id())); + //std::cout << "WITHSMP " + //<< id_to_index(p1.id()) << " " + //<< id_to_index(p2.id()) << " " + //<< p1.is_ghost() << " " + //<< p2.is_ghost() << " " + //<< p1.id() << " " + //<< p2.id() << std::endl; //<< p1.pos() << " " //<< p2.pos() << "\n"; - }; + }; - cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion);*/ + cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); + } } else { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); @@ -507,8 +491,13 @@ void cabana_short_range( defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) unique_particles, #endif - bonded_ias, nonbonded_ias, box_geo, local_force, local_torque, - local_virial, aosoa, + bonded_ias, nonbonded_ias, box_geo, aosoa, local_force, +#ifdef ROTATION + local_torque, +#endif +#ifdef NPT + local_virial, +#endif #ifdef COLLISION_DETECTION *collision_detection, #endif @@ -523,28 +512,36 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); #endif + if (1) { verlet_list = create_verlet_list(max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, + //sequential_particles, verlet_criterion, first_neighbor_kernel, cell_structure); -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List by Cabana"); -#endif - } else { - //{ -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - calc Force"); -#endif - /* - using neighbor_list = Cabana::NeighborList; + /*using neighbor_list = Cabana::NeighborList; std::vector> interaction_pairs; for (int i = 0; i < number_of_unique_particles; ++i) { for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = neighbor_list::getNeighbor(verlet_list, i, n); - interaction_pairs.emplace_back(i, j); + //interaction_pairs.emplace_back(i, j); + std::cout << "*Cabana* " + << i << " " + << j << " " + << aosoa.ghost(i) << " " + << aosoa.ghost(j) << " " + << aosoa.id(i) << " " + << aosoa.id(j) << "\n"; } + }*/ } - */ +#ifdef CALIPER + CALI_MARK_END("Cabana - Verlet List by Cabana"); +#endif + } else { + //{ +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - calc Force"); +#endif /* Kokkos::parallel_for("ForceLoop", Kokkos::RangePolicy<>(0, interaction_pairs.size()), KOKKOS_LAMBDA(int idx) { auto i = @@ -566,8 +563,9 @@ void cabana_short_range( // Save data for next iteration if we just rebuilt if (rebuild) { - CabanaData new_data(verlet_list, unique_particles, - unique_particles.size()); + //CabanaData new_data(verlet_list, unique_particles, + // unique_particles.size()); + CabanaData new_data(verlet_list, unique_particles, max_id); cell_structure.set_cabana_data(std::make_unique(new_data)); } diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index fd248b7e783..3c964ebf4e4 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -192,10 +192,11 @@ using ListType = Cabana::CustomVerletList; template ListType create_verlet_list(double const max_cutoff, int const max_counts, - AoSoA_pack aosoa, - std::vector unique_particles, + AoSoA_pack &aosoa, + std::vector &unique_particles, + //std::vector &unique_particles, VerletCriterion const &verlet_criterion, - Kernel first_neighbor_kernel, + Kernel &first_neighbor_kernel, CellStructure &cell_structure) { // Creating LinkedCellList and VerletList: // Box Properties @@ -256,7 +257,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, Kokkos::View bin_offset("bin_offset", total_bins); Kokkos::View bin_size("bin_size", total_bins); Kokkos::View original_idx( - "original_idx", unique_particles.size()); + "original_idx", aosoa.position.size()); set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, bin_offset, bin_size, original_idx); auto const particle_bins = cell_list.getParticleBins(); @@ -292,7 +293,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, &unique_particles, &verlet_criterion, &distance_function, &verlet_list, &first_neighbor_kernel] //, thread_id] - (Particle * p1, int ii, int id_i, int cell_offset, int cell_size) { + (Particle* p1, int ii, int id_i, int cell_offset, int cell_size) { for (int j = cell_offset; j < cell_offset + cell_size; ++j) { // int ii = cell_list.permutation(i); // debug // int jj = j; @@ -309,15 +310,19 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto p2 = unique_particles.at(jj); // auto p2 = cell_structure.get_local_particle(id_j); if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(ii, jj); +#ifdef EXCLUSIONS + verlet_list.addNeighbor(std::min(ii, jj), std::max(ii, jj)); +#else + verlet_list.addNeighbor(ii, jj); +#endif // verlet_list.addNeighborNonAtomic(thread_id, ii, jj); - /*std::cout << "*Cabana* " - << i << " " - << j << " " - << id_i << " " - << id_j << " " + /*std::cout << "*Ca* " + << ii << " " + << jj << " " << aosoa_ghost(ii) << " " << aosoa_ghost(jj) << " " + << id_i << " " + << id_j << "\n"; << cid_i << " " << cid_j << " " << aosoa.position(ii, 0) << ", " diff --git a/testsuite/python/exclusions.py b/testsuite/python/exclusions.py index 3267a0a95f5..1fc8ca427d6 100644 --- a/testsuite/python/exclusions.py +++ b/testsuite/python/exclusions.py @@ -57,9 +57,8 @@ def test_transfer(self): p0.exclusions = [1, 2, 3] - i = 0 for _ in range(15): - i += 1 + print('run') self.system.integrator.run(100) self.assertEqual(list(p0.exclusions), [1, 2, 3]) From 3fc4140faedb6069d6195f62f2cf083439311013 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 30 Jun 2025 17:23:57 +0200 Subject: [PATCH 40/94] Modified creation of VerletList --- src/core/cabana_data.hpp | 14 +++--- src/core/custom_verlet_list.hpp | 38 +++++++++++++++ src/core/short_range_cabana.hpp | 86 +++++++++++++++++++++++++-------- src/core/verlet_list_loop.hpp | 9 ++-- 4 files changed, 118 insertions(+), 29 deletions(-) diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index cc82c58e427..d10d5dd7bdd 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -37,18 +37,20 @@ class CabanaData { private: ListType verlet_list; std::vector unique_particles; - int max_id; + //int max_id; public: CabanaData() = default; - CabanaData(ListType verlet_list, std::vector unique_particles, - int max_id) - : verlet_list(verlet_list), unique_particles(unique_particles), - max_id(max_id) {} + CabanaData(ListType verlet_list, std::vector unique_particles) + : verlet_list(verlet_list), unique_particles(unique_particles) {} + //CabanaData(ListType verlet_list, std::vector unique_particles, + // int max_id) + // : verlet_list(verlet_list), unique_particles(unique_particles), + // max_id(max_id) {} ListType get_verlet_list() const { return verlet_list; } int get_index() const { return unique_particles.size(); } - int get_max_id() const { return max_id; } + //int get_max_id() const { return max_id; } std::vector get_unique_particles() const { return unique_particles; } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 85c3487fd8f..f907db4cc83 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -45,6 +45,7 @@ class CustomVerletList virtual ~CustomVerletList() {}; private: + Kokkos::View max_thread; Kokkos::View counts_thread; Kokkos::View neighbors_thread; @@ -61,6 +62,11 @@ class CustomVerletList neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, max_neigh); + max_thread = Kokkos::View("max_thread", thread_number); + for (int tid = 0; tid < thread_number; ++tid) { + max_thread(tid) = 1; + } + /* counts_thread = Kokkos::View( "num_neighbors", thread_number, num_particles); neighbors_thread = Kokkos::View( @@ -72,6 +78,7 @@ class CustomVerletList counts_thread(tid, i) = 0; } }); + */ } // Method to dynamically expand the size of max_neighbors @@ -100,6 +107,7 @@ class CustomVerletList // Method to add a neighbor KOKKOS_INLINE_FUNCTION +#ifdef EXCLUSIONS void addNeighbor(const int pid, const int nid) { std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); if (count >= neighbors.extent(1)) { @@ -109,6 +117,19 @@ class CustomVerletList } neighbors(pid, count) = nid; } +#else + void addNeighbor(const int tid, int pid, int nid) { + if (counts(pid) + 1 > max_thread(tid)) std::swap(pid, nid); + std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); + if (count >= neighbors.extent(1)) { + // expandMaxNeighbors(neighbors.extent(1) * 2); + throw std::runtime_error( + "Number of count is larger than VerletList size."); + } + neighbors(pid, count) = nid; + if (counts(pid) > max_thread(tid)) max_thread(tid) = counts(pid); + } +#endif // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION @@ -122,6 +143,7 @@ class CustomVerletList } // Reduction of counts and neighbor in all threads + KOKKOS_INLINE_FUNCTION void reduction() { // Kokkos::RangePolicy policy(0, counts.extent(0)); int thread_number = counts_thread.extent(0); @@ -142,6 +164,22 @@ class CustomVerletList }); Kokkos::fence(); } + + // Find max counts + KOKKOS_INLINE_FUNCTION + std::size_t get_max_counts() { + std::size_t max_counts = 0; + std::size_t ave_counts = 0; + for (int pid = 0; pid < counts.extent(0); ++pid) { + if (max_counts < counts(pid)) max_counts = counts(pid); + ave_counts += counts(pid); + } + if (counts.extent(0) != 0) { + std::cout << "max:" << max_counts + << " ave:" << ave_counts/counts.extent(0) << std::endl; + } + return max_counts; + } }; template diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 25c58720d27..10b45264fb3 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -136,7 +136,7 @@ void cabana_short_range( std::vector unique_particles; //std::vector sequential_particles; int index = 0; - int max_id = 0; + //int max_id = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); //if (rank == 0) { @@ -157,7 +157,7 @@ void cabana_short_range( for (auto &p : particles) { if (cell_structure.get_local_particle(p.id())) { - if (p.id() > max_id) max_id = p.id(); + //if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); //sequential_particles.emplace_back(p); @@ -168,7 +168,7 @@ void cabana_short_range( for (auto &p : ghost_particles) { if (not registered_index.contains(p.id())) { if (cell_structure.get_local_particle(p.id())) { - if (p.id() > max_id) max_id = p.id(); + //if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); //sequential_particles.emplace_back(p); @@ -180,11 +180,11 @@ void cabana_short_range( // If we do not rebuild we can use the saved map index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); - max_id = saved_data.get_max_id(); - //sequential_particles.reserve(unique_particles.size()); - //for (Particle * ptr : unique_particles) { - // sequential_particles.emplace_back(*ptr); - //} + //max_id = saved_data.get_max_id(); + /*sequential_particles.reserve(unique_particles.size()); + for (Particle * ptr : unique_particles) { + sequential_particles.emplace_back(*ptr); + }*/ } int number_of_unique_particles = index; @@ -200,23 +200,68 @@ void cabana_short_range( #endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); + auto slice_position = Cabana::slice<0>(particle_storage); + auto slice_charge = Cabana::slice<1>(particle_storage); + auto slice_id = Cabana::slice<2>(particle_storage); + auto slice_type = Cabana::slice<3>(particle_storage); + auto slice_ghost = Cabana::slice<4>(particle_storage); // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); Kokkos::RangePolicy allocation_policy( 0, number_of_unique_particles); - Kokkos::View id_to_index("id_to_index", max_id + 1); - //Kokkos::parallel_for("allocation", allocation_policy, - // [&unique_particles, &box_l, &aosoa, &id_to_index](int p_id) { + //Kokkos::View id_to_index("id_to_index", max_id + 1); + //Kokkos::View device_particles("particles", number_of_unique_particles); + //for (int i = 0; i < number_of_unique_particles; ++i) { + // device_particles(i) = *unique_particles[i]; + //} + + /*Kokkos::parallel_for("allocation", allocation_policy, + // [&unique_particles, &box_l, &aosoa](int p_id) { for (int p_id = 0; p_id < number_of_unique_particles; ++p_id) { //auto thread_id = omp_get_thread_num(); auto p = *unique_particles[p_id]; //auto p = sequential_particles[p_id]; // auto p = device_particles(p_id); write_particle(p, p_id, aosoa, box_l); - id_to_index(p.id()) = p_id; + //id_to_index(p.id()) = p_id; + } + //});*/ + /*using policy_type = Kokkos::TeamPolicy; + int num_particles = number_of_unique_particles; + int num_blocks = (num_particles + vector_length - 1) / vector_length; + Kokkos::parallel_for( + "AoSoA Write", + policy_type(num_blocks, Kokkos::AUTO), + [&unique_particles, &slice_position, &slice_charge, &slice_id, &slice_type, slice_ghost, &box_l, number_of_unique_particles] + (const policy_type::member_type& team_member) { + const int block = team_member.league_rank(); + const int start = block * vector_length; + const int end = start + vector_length; + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, vector_length), [=](const int i) { + const int p_id = start + i; + + if (p_id >= number_of_unique_particles) return;*/ + //Kokkos::parallel_for("convert_to_aosoa", allocation_policy, + // [&unique_particles, &slice_position, &slice_charge, &slice_id, + // &slice_type, slice_ghost, &box_l] (const int p_id) { + for (int p_id = 0; p_id < unique_particles.size(); ++p_id) { + Particle p = *unique_particles.at(p_id); + + auto pos = p.pos(); + for (int d = 0; d < 3; ++d) { + double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + slice_position(p_id, d) = wrapped; + } + + slice_charge(p_id) = p.q(); // charge + slice_id(p_id) = p.id(); // id + slice_type(p_id) = p.type(); // type + slice_ghost(p_id) = p.is_ghost(); // ghost } + // }); //}); Kokkos::fence(); @@ -445,7 +490,7 @@ void cabana_short_range( if (at_steepest_descent) { max_prefactor = 8; } else { - max_prefactor = 6; + max_prefactor = 4; } max_counts = static_cast(std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); @@ -459,11 +504,12 @@ void cabana_short_range( } //std::cout << "max_counts:" << max_counts << " " << max_cutoff << std::endl; if (rebuild) { // Legacy Velert List - if (0) { - verlet_list = + /*verlet_list = ListType(aosoa.position, 0, aosoa.position.size(), max_counts, num_threads); auto kernel = [&](Particle const &p1, Particle const &p2) { - verlet_list.addNeighbor(id_to_index(p1.id()), + auto thread_id = omp_get_thread_num(); + verlet_list.addNeighbor(thread_id, + id_to_index(p1.id()), id_to_index(p2.id())); //std::cout << "WITHSMP " //<< id_to_index(p1.id()) << " " @@ -477,7 +523,7 @@ void cabana_short_range( }; cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); - } + */ } else { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); @@ -533,6 +579,7 @@ void cabana_short_range( << aosoa.id(j) << "\n"; } }*/ + //verlet_list.get_max_counts(); } #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List by Cabana"); @@ -549,6 +596,7 @@ void cabana_short_range( first_neighbor_kernel(i, j); }); */ + //verlet_list.get_max_counts(); Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, @@ -565,7 +613,7 @@ void cabana_short_range( if (rebuild) { //CabanaData new_data(verlet_list, unique_particles, // unique_particles.size()); - CabanaData new_data(verlet_list, unique_particles, max_id); + CabanaData new_data(verlet_list, unique_particles); cell_structure.set_cabana_data(std::make_unique(new_data)); } @@ -591,7 +639,7 @@ void cabana_short_range( ty += local_torque(tid, i, 1); tz += local_torque(tid, i, 2); } - auto &p = unique_particles[i]; + auto &p = unique_particles.at(i); // auto p = // cell_structure.get_local_particle(aosoa.id(i)); p->force() += Utils::Vector3d{fx, fy, fz}; diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 3c964ebf4e4..10288fe4e41 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -285,14 +285,14 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, &distance_function, &verlet_list, &first_neighbor_kernel](const int pair_cell_i) { - // auto thread_id = omp_get_thread_num(); + auto thread_id = omp_get_thread_num(); int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); auto verlet_kernel = [&original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, &distance_function, &verlet_list, - &first_neighbor_kernel] //, thread_id] + &first_neighbor_kernel, thread_id] (Particle* p1, int ii, int id_i, int cell_offset, int cell_size) { for (int j = cell_offset; j < cell_offset + cell_size; ++j) { // int ii = cell_list.permutation(i); // debug @@ -312,10 +312,11 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { #ifdef EXCLUSIONS verlet_list.addNeighbor(std::min(ii, jj), std::max(ii, jj)); + //verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), std::max(ii, jj)); #else - verlet_list.addNeighbor(ii, jj); + verlet_list.addNeighbor(thread_id, ii, jj); + //verlet_list.addNeighborNonAtomic(thread_id, ii, jj); #endif - // verlet_list.addNeighborNonAtomic(thread_id, ii, jj); /*std::cout << "*Ca* " << ii << " " << jj << " " From e344e6ea8cc8990d2f0d86b9a9a5794b480d3f58 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 30 Jun 2025 17:26:40 +0200 Subject: [PATCH 41/94] Formatting --- src/core/cabana_data.hpp | 12 +-- src/core/custom_verlet_list.hpp | 11 +- src/core/short_range_cabana.hpp | 171 ++++++++++++++++---------------- src/core/verlet_list_loop.hpp | 25 ++--- 4 files changed, 111 insertions(+), 108 deletions(-) diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index d10d5dd7bdd..b22549eb7b9 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -37,20 +37,20 @@ class CabanaData { private: ListType verlet_list; std::vector unique_particles; - //int max_id; + // int max_id; public: CabanaData() = default; CabanaData(ListType verlet_list, std::vector unique_particles) : verlet_list(verlet_list), unique_particles(unique_particles) {} - //CabanaData(ListType verlet_list, std::vector unique_particles, - // int max_id) - // : verlet_list(verlet_list), unique_particles(unique_particles), - // max_id(max_id) {} + // CabanaData(ListType verlet_list, std::vector unique_particles, + // int max_id) + // : verlet_list(verlet_list), unique_particles(unique_particles), + // max_id(max_id) {} ListType get_verlet_list() const { return verlet_list; } int get_index() const { return unique_particles.size(); } - //int get_max_id() const { return max_id; } + // int get_max_id() const { return max_id; } std::vector get_unique_particles() const { return unique_particles; } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index f907db4cc83..9832b7985ae 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -119,7 +119,8 @@ class CustomVerletList } #else void addNeighbor(const int tid, int pid, int nid) { - if (counts(pid) + 1 > max_thread(tid)) std::swap(pid, nid); + if (counts(pid) + 1 > max_thread(tid)) + std::swap(pid, nid); std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); if (count >= neighbors.extent(1)) { // expandMaxNeighbors(neighbors.extent(1) * 2); @@ -127,7 +128,8 @@ class CustomVerletList "Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; - if (counts(pid) > max_thread(tid)) max_thread(tid) = counts(pid); + if (counts(pid) > max_thread(tid)) + max_thread(tid) = counts(pid); } #endif @@ -171,12 +173,13 @@ class CustomVerletList std::size_t max_counts = 0; std::size_t ave_counts = 0; for (int pid = 0; pid < counts.extent(0); ++pid) { - if (max_counts < counts(pid)) max_counts = counts(pid); + if (max_counts < counts(pid)) + max_counts = counts(pid); ave_counts += counts(pid); } if (counts.extent(0) != 0) { std::cout << "max:" << max_counts - << " ave:" << ave_counts/counts.extent(0) << std::endl; + << " ave:" << ave_counts / counts.extent(0) << std::endl; } return max_counts; } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 10b45264fb3..2706e23791a 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -134,15 +134,15 @@ void cabana_short_range( #endif std::unordered_set registered_index{}; std::vector unique_particles; - //std::vector sequential_particles; + // std::vector sequential_particles; int index = 0; - //int max_id = 0; + // int max_id = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); - //if (rank == 0) { - // std::cout << "For CABANA rebuild " << rebuild + // if (rank == 0) { + // std::cout << "For CABANA rebuild " << rebuild // << " " << Kokkos::OpenMP::concurrency() << std::endl; - //} + // } CabanaData saved_data; @@ -157,10 +157,10 @@ void cabana_short_range( for (auto &p : particles) { if (cell_structure.get_local_particle(p.id())) { - //if (p.id() > max_id) max_id = p.id(); + // if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); - //sequential_particles.emplace_back(p); + // sequential_particles.emplace_back(p); index++; } } @@ -168,10 +168,10 @@ void cabana_short_range( for (auto &p : ghost_particles) { if (not registered_index.contains(p.id())) { if (cell_structure.get_local_particle(p.id())) { - //if (p.id() > max_id) max_id = p.id(); + // if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); - //sequential_particles.emplace_back(p); + // sequential_particles.emplace_back(p); index++; } } @@ -180,10 +180,10 @@ void cabana_short_range( // If we do not rebuild we can use the saved map index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); - //max_id = saved_data.get_max_id(); + // max_id = saved_data.get_max_id(); /*sequential_particles.reserve(unique_particles.size()); for (Particle * ptr : unique_particles) { - sequential_particles.emplace_back(*ptr); + sequential_particles.emplace_back(*ptr); }*/ } @@ -201,22 +201,23 @@ void cabana_short_range( Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); auto slice_position = Cabana::slice<0>(particle_storage); - auto slice_charge = Cabana::slice<1>(particle_storage); - auto slice_id = Cabana::slice<2>(particle_storage); - auto slice_type = Cabana::slice<3>(particle_storage); - auto slice_ghost = Cabana::slice<4>(particle_storage); + auto slice_charge = Cabana::slice<1>(particle_storage); + auto slice_id = Cabana::slice<2>(particle_storage); + auto slice_type = Cabana::slice<3>(particle_storage); + auto slice_ghost = Cabana::slice<4>(particle_storage); // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); Kokkos::RangePolicy allocation_policy( 0, number_of_unique_particles); - //Kokkos::View id_to_index("id_to_index", max_id + 1); - //Kokkos::View device_particles("particles", number_of_unique_particles); - //for (int i = 0; i < number_of_unique_particles; ++i) { - // device_particles(i) = *unique_particles[i]; - //} - + // Kokkos::View id_to_index("id_to_index", max_id + 1); + // Kokkos::View device_particles("particles", + // number_of_unique_particles); for (int i = 0; i < + // number_of_unique_particles; ++i) { + // device_particles(i) = *unique_particles[i]; + // } + /*Kokkos::parallel_for("allocation", allocation_policy, // [&unique_particles, &box_l, &aosoa](int p_id) { for (int p_id = 0; p_id < number_of_unique_particles; ++p_id) { @@ -234,32 +235,32 @@ void cabana_short_range( Kokkos::parallel_for( "AoSoA Write", policy_type(num_blocks, Kokkos::AUTO), - [&unique_particles, &slice_position, &slice_charge, &slice_id, &slice_type, slice_ghost, &box_l, number_of_unique_particles] - (const policy_type::member_type& team_member) { - const int block = team_member.league_rank(); - const int start = block * vector_length; - const int end = start + vector_length; - - Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, vector_length), [=](const int i) { - const int p_id = start + i; - - if (p_id >= number_of_unique_particles) return;*/ - //Kokkos::parallel_for("convert_to_aosoa", allocation_policy, - // [&unique_particles, &slice_position, &slice_charge, &slice_id, - // &slice_type, slice_ghost, &box_l] (const int p_id) { + [&unique_particles, &slice_position, &slice_charge, &slice_id, + &slice_type, slice_ghost, &box_l, number_of_unique_particles] (const + policy_type::member_type& team_member) { const int block = + team_member.league_rank(); const int start = block * vector_length; const + int end = start + vector_length; + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, + vector_length), [=](const int i) { const int p_id = start + i; + + if (p_id >= number_of_unique_particles) return;*/ + // Kokkos::parallel_for("convert_to_aosoa", allocation_policy, + // [&unique_particles, &slice_position, &slice_charge, &slice_id, + // &slice_type, slice_ghost, &box_l] (const int p_id) { for (int p_id = 0; p_id < unique_particles.size(); ++p_id) { - Particle p = *unique_particles.at(p_id); - - auto pos = p.pos(); - for (int d = 0; d < 3; ++d) { - double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; - slice_position(p_id, d) = wrapped; - } - - slice_charge(p_id) = p.q(); // charge - slice_id(p_id) = p.id(); // id - slice_type(p_id) = p.type(); // type - slice_ghost(p_id) = p.is_ghost(); // ghost + Particle p = *unique_particles.at(p_id); + + auto pos = p.pos(); + for (int d = 0; d < 3; ++d) { + double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + slice_position(p_id, d) = wrapped; + } + + slice_charge(p_id) = p.q(); // charge + slice_id(p_id) = p.id(); // id + slice_type(p_id) = p.type(); // type + slice_ghost(p_id) = p.is_ghost(); // ghost } // }); //}); @@ -321,8 +322,7 @@ void cabana_short_range( #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, - AoSoA_pack &aosoa_, + const BoxGeometry &box_geo_, AoSoA_pack &aosoa_, Kokkos::View local_force_, #ifdef ROTATION Kokkos::View local_torque_, @@ -351,11 +351,9 @@ void cabana_short_range( unique_particles(unique_particles_), #endif bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - box_geo(box_geo_), - aosoa(aosoa_), - local_force(local_force_), + box_geo(box_geo_), aosoa(aosoa_), local_force(local_force_), #ifdef ROTATION - local_torque(local_torque_), + local_torque(local_torque_), #endif #ifdef NPT local_virial(local_virial_), @@ -488,12 +486,12 @@ void cabana_short_range( } else { int max_prefactor; if (at_steepest_descent) { - max_prefactor = 8; + max_prefactor = 8; } else { - max_prefactor = 4; + max_prefactor = 4; } - max_counts = - static_cast(std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); + max_counts = static_cast( + std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); } int threshold_num = 8; #ifdef COLLISION_DETECTION @@ -502,14 +500,15 @@ void cabana_short_range( if (max_counts < threshold_num) { max_counts = std::min(threshold_num, number_of_unique_particles); } - //std::cout << "max_counts:" << max_counts << " " << max_cutoff << std::endl; + // std::cout << "max_counts:" << max_counts << " " << max_cutoff << + // std::endl; if (rebuild) { // Legacy Velert List /*verlet_list = - ListType(aosoa.position, 0, aosoa.position.size(), max_counts, num_threads); - auto kernel = [&](Particle const &p1, Particle const &p2) { + ListType(aosoa.position, 0, aosoa.position.size(), max_counts, + num_threads); auto kernel = [&](Particle const &p1, Particle const &p2) { auto thread_id = omp_get_thread_num(); verlet_list.addNeighbor(thread_id, - id_to_index(p1.id()), + id_to_index(p1.id()), id_to_index(p2.id())); //std::cout << "WITHSMP " //<< id_to_index(p1.id()) << " " @@ -539,7 +538,7 @@ void cabana_short_range( #endif bonded_ias, nonbonded_ias, box_geo, aosoa, local_force, #ifdef ROTATION - local_torque, + local_torque, #endif #ifdef NPT local_virial, @@ -559,33 +558,33 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); #endif if (1) { - verlet_list = create_verlet_list(max_cutoff, max_counts, aosoa, - unique_particles, verlet_criterion, - //sequential_particles, verlet_criterion, - first_neighbor_kernel, cell_structure); - /*using neighbor_list = Cabana::NeighborList; - std::vector> interaction_pairs; - - for (int i = 0; i < number_of_unique_particles; ++i) { - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - //interaction_pairs.emplace_back(i, j); - std::cout << "*Cabana* " - << i << " " - << j << " " - << aosoa.ghost(i) << " " - << aosoa.ghost(j) << " " - << aosoa.id(i) << " " - << aosoa.id(j) << "\n"; - } - }*/ - //verlet_list.get_max_counts(); + verlet_list = create_verlet_list( + max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, + // sequential_particles, verlet_criterion, + first_neighbor_kernel, cell_structure); + /*using neighbor_list = Cabana::NeighborList; + std::vector> interaction_pairs; + + for (int i = 0; i < number_of_unique_particles; ++i) { + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + //interaction_pairs.emplace_back(i, j); + std::cout << "*Cabana* " + << i << " " + << j << " " + << aosoa.ghost(i) << " " + << aosoa.ghost(j) << " " + << aosoa.id(i) << " " + << aosoa.id(j) << "\n"; + } + }*/ + // verlet_list.get_max_counts(); } #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List by Cabana"); #endif } else { - //{ + //{ #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); #endif @@ -596,7 +595,7 @@ void cabana_short_range( first_neighbor_kernel(i, j); }); */ - //verlet_list.get_max_counts(); + // verlet_list.get_max_counts(); Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, @@ -611,8 +610,8 @@ void cabana_short_range( // Save data for next iteration if we just rebuilt if (rebuild) { - //CabanaData new_data(verlet_list, unique_particles, - // unique_particles.size()); + // CabanaData new_data(verlet_list, unique_particles, + // unique_particles.size()); CabanaData new_data(verlet_list, unique_particles); cell_structure.set_cabana_data(std::make_unique(new_data)); } diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 10288fe4e41..9fd2e05196c 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -194,7 +194,7 @@ template ListType create_verlet_list(double const max_cutoff, int const max_counts, AoSoA_pack &aosoa, std::vector &unique_particles, - //std::vector &unique_particles, + // std::vector &unique_particles, VerletCriterion const &verlet_criterion, Kernel &first_neighbor_kernel, CellStructure &cell_structure) { @@ -256,8 +256,8 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // Offset particle id and the number of particle in specific cell Kokkos::View bin_offset("bin_offset", total_bins); Kokkos::View bin_size("bin_size", total_bins); - Kokkos::View original_idx( - "original_idx", aosoa.position.size()); + Kokkos::View original_idx("original_idx", + aosoa.position.size()); set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, bin_offset, bin_size, original_idx); auto const particle_bins = cell_list.getParticleBins(); @@ -289,11 +289,11 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); - auto verlet_kernel = [&original_idx, &aosoa_id, &aosoa_ghost, - &unique_particles, &verlet_criterion, - &distance_function, &verlet_list, - &first_neighbor_kernel, thread_id] - (Particle* p1, int ii, int id_i, int cell_offset, int cell_size) { + auto verlet_kernel = + [&original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, + &verlet_criterion, &distance_function, &verlet_list, + &first_neighbor_kernel, thread_id](Particle *p1, int ii, int id_i, + int cell_offset, int cell_size) { for (int j = cell_offset; j < cell_offset + cell_size; ++j) { // int ii = cell_list.permutation(i); // debug // int jj = j; @@ -311,11 +311,12 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // auto p2 = cell_structure.get_local_particle(id_j); if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { #ifdef EXCLUSIONS - verlet_list.addNeighbor(std::min(ii, jj), std::max(ii, jj)); - //verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), std::max(ii, jj)); + verlet_list.addNeighbor(std::min(ii, jj), std::max(ii, jj)); + // verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), + // std::max(ii, jj)); #else - verlet_list.addNeighbor(thread_id, ii, jj); - //verlet_list.addNeighborNonAtomic(thread_id, ii, jj); + verlet_list.addNeighbor(thread_id, ii, jj); + // verlet_list.addNeighborNonAtomic(thread_id, ii, jj); #endif /*std::cout << "*Ca* " << ii << " " From dee9d2089b16b6eacca5e8cafc805aa4d7a23a10 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 1 Jul 2025 18:00:03 +0200 Subject: [PATCH 42/94] Added load balancing for verlet list --- src/core/custom_verlet_list.hpp | 12 ----- src/core/exclusions.hpp | 10 +++-- src/core/short_range_cabana.hpp | 80 ++++++++++----------------------- src/core/verlet_list_loop.hpp | 13 +++--- testsuite/python/exclusions.py | 1 - 5 files changed, 37 insertions(+), 79 deletions(-) diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 9832b7985ae..929236f6dfe 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -107,17 +107,6 @@ class CustomVerletList // Method to add a neighbor KOKKOS_INLINE_FUNCTION -#ifdef EXCLUSIONS - void addNeighbor(const int pid, const int nid) { - std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); - if (count >= neighbors.extent(1)) { - // expandMaxNeighbors(neighbors.extent(1) * 2); - throw std::runtime_error( - "Number of count is larger than VerletList size."); - } - neighbors(pid, count) = nid; - } -#else void addNeighbor(const int tid, int pid, int nid) { if (counts(pid) + 1 > max_thread(tid)) std::swap(pid, nid); @@ -131,7 +120,6 @@ class CustomVerletList if (counts(pid) > max_thread(tid)) max_thread(tid) = counts(pid); } -#endif // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION diff --git a/src/core/exclusions.hpp b/src/core/exclusions.hpp index b126fcc732b..6994bbf8d0c 100644 --- a/src/core/exclusions.hpp +++ b/src/core/exclusions.hpp @@ -33,10 +33,14 @@ * calculated. */ inline bool do_nonbonded(Particle const &p1, Particle const &p2) { - /* check for particle 2 in particle 1's exclusion list. The exclusion list is - * symmetric, so this is sufficient. */ - return std::ranges::none_of( + /* check for particle 2 in particle 1's exclusion list. The exclusion list should + * be symmetric, so this is sufficient. */ + /* However. in present implementation, the exclusion list is not symmetric.*/ + bool p1_p2 = std::ranges::none_of( p1.exclusions(), [p2_id = p2.id()](int id) { return id == p2_id; }); + bool p2_p1 = std::ranges::none_of( + p2.exclusions(), [p1_id = p1.id()](int id) { return id == p1_id; }); + return (p1_p2 or p2_p1); } /** Remove exclusion from particle if possible */ diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 2706e23791a..8e0b169ac61 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -181,10 +181,6 @@ void cabana_short_range( index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); // max_id = saved_data.get_max_id(); - /*sequential_particles.reserve(unique_particles.size()); - for (Particle * ptr : unique_particles) { - sequential_particles.emplace_back(*ptr); - }*/ } int number_of_unique_particles = index; @@ -206,64 +202,36 @@ void cabana_short_range( auto slice_type = Cabana::slice<3>(particle_storage); auto slice_ghost = Cabana::slice<4>(particle_storage); // particle properties are defined in aosoa_pack.hpp + particle_storage.resize(number_of_unique_particles); auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); - Kokkos::RangePolicy allocation_policy( - 0, number_of_unique_particles); - // Kokkos::View id_to_index("id_to_index", max_id + 1); - // Kokkos::View device_particles("particles", - // number_of_unique_particles); for (int i = 0; i < - // number_of_unique_particles; ++i) { - // device_particles(i) = *unique_particles[i]; - // } - - /*Kokkos::parallel_for("allocation", allocation_policy, - // [&unique_particles, &box_l, &aosoa](int p_id) { - for (int p_id = 0; p_id < number_of_unique_particles; ++p_id) { - //auto thread_id = omp_get_thread_num(); - auto p = *unique_particles[p_id]; - //auto p = sequential_particles[p_id]; - // auto p = device_particles(p_id); - write_particle(p, p_id, aosoa, box_l); - //id_to_index(p.id()) = p_id; - } - //});*/ - /*using policy_type = Kokkos::TeamPolicy; - int num_particles = number_of_unique_particles; - int num_blocks = (num_particles + vector_length - 1) / vector_length; + using policy_type = Kokkos::TeamPolicy; + int team_size = 1; + int league_size = (unique_particles.size() + team_size - 1) / team_size; Kokkos::parallel_for( "AoSoA Write", - policy_type(num_blocks, Kokkos::AUTO), - [&unique_particles, &slice_position, &slice_charge, &slice_id, - &slice_type, slice_ghost, &box_l, number_of_unique_particles] (const - policy_type::member_type& team_member) { const int block = - team_member.league_rank(); const int start = block * vector_length; const - int end = start + vector_length; - - Kokkos::parallel_for(Kokkos::TeamThreadRange(team_member, - vector_length), [=](const int i) { const int p_id = start + i; - - if (p_id >= number_of_unique_particles) return;*/ - // Kokkos::parallel_for("convert_to_aosoa", allocation_policy, - // [&unique_particles, &slice_position, &slice_charge, &slice_id, - // &slice_type, slice_ghost, &box_l] (const int p_id) { - for (int p_id = 0; p_id < unique_particles.size(); ++p_id) { - Particle p = *unique_particles.at(p_id); - - auto pos = p.pos(); - for (int d = 0; d < 3; ++d) { - double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; - slice_position(p_id, d) = wrapped; - } + policy_type(league_size, team_size), + [&unique_particles, &aosoa, &box_l, team_size, number_of_unique_particles] + (const policy_type::member_type& team_member) { - slice_charge(p_id) = p.q(); // charge - slice_id(p_id) = p.id(); // id - slice_type(p_id) = p.type(); // type - slice_ghost(p_id) = p.is_ghost(); // ghost - } - // }); - //}); + int p_id = team_member.league_rank() * team_size + team_member.team_rank(); + if (p_id >= number_of_unique_particles) return; + + Particle p = *unique_particles.at(p_id); + + write_particle(p, p_id, aosoa, box_l); + /*auto pos = p.pos(); + for (int d = 0; d < 3; ++d) { + double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + aosoa.position(p_id, d) = wrapped; + } + + aosoa.charge(p_id) = p.q(); // charge + aosoa.id(p_id) = p.id(); // id + aosoa.type(p_id) = p.type(); // type + aosoa.ghost(p_id) = p.is_ghost(); // ghost*/ + }); Kokkos::fence(); Kokkos::View local_force( diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 9fd2e05196c..7a4ebe5574f 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -299,6 +299,11 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // int jj = j; int jj = original_idx(j); int id_j = aosoa_id(jj); + /*if (id_i < id_j) { + if (aosoa_ghost(ii)) continue; + } else { + if (aosoa_ghost(jj)) continue; + }*/ if (aosoa_ghost(ii) or aosoa_ghost(jj)) { if (((id_i < id_j) and aosoa_ghost(ii)) or ((id_i > id_j) and aosoa_ghost(jj))) { @@ -310,14 +315,8 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto p2 = unique_particles.at(jj); // auto p2 = cell_structure.get_local_particle(id_j); if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { -#ifdef EXCLUSIONS - verlet_list.addNeighbor(std::min(ii, jj), std::max(ii, jj)); - // verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), - // std::max(ii, jj)); -#else verlet_list.addNeighbor(thread_id, ii, jj); - // verlet_list.addNeighborNonAtomic(thread_id, ii, jj); -#endif + // verlet_list.addNeighborNonAtomic(thread_id, ii, jj); /*std::cout << "*Ca* " << ii << " " << jj << " " diff --git a/testsuite/python/exclusions.py b/testsuite/python/exclusions.py index 1fc8ca427d6..4cc1aaccb5f 100644 --- a/testsuite/python/exclusions.py +++ b/testsuite/python/exclusions.py @@ -58,7 +58,6 @@ def test_transfer(self): p0.exclusions = [1, 2, 3] for _ in range(15): - print('run') self.system.integrator.run(100) self.assertEqual(list(p0.exclusions), [1, 2, 3]) From 87dc45252bc01fdfbfd780bb9d991f11d6336ae7 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 1 Jul 2025 18:10:17 +0200 Subject: [PATCH 43/94] Formatting --- src/core/exclusions.hpp | 4 +-- src/core/short_range_cabana.hpp | 45 +++++++++++++++++---------------- src/core/verlet_list_loop.hpp | 10 ++++---- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/core/exclusions.hpp b/src/core/exclusions.hpp index 6994bbf8d0c..2ed62db6380 100644 --- a/src/core/exclusions.hpp +++ b/src/core/exclusions.hpp @@ -33,8 +33,8 @@ * calculated. */ inline bool do_nonbonded(Particle const &p1, Particle const &p2) { - /* check for particle 2 in particle 1's exclusion list. The exclusion list should - * be symmetric, so this is sufficient. */ + /* check for particle 2 in particle 1's exclusion list. The exclusion list + * should be symmetric, so this is sufficient. */ /* However. in present implementation, the exclusion list is not symmetric.*/ bool p1_p2 = std::ranges::none_of( p1.exclusions(), [p2_id = p2.id()](int id) { return id == p2_id; }); diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 8e0b169ac61..eeebe38b8de 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -209,29 +209,30 @@ void cabana_short_range( using policy_type = Kokkos::TeamPolicy; int team_size = 1; int league_size = (unique_particles.size() + team_size - 1) / team_size; - Kokkos::parallel_for( - "AoSoA Write", - policy_type(league_size, team_size), - [&unique_particles, &aosoa, &box_l, team_size, number_of_unique_particles] - (const policy_type::member_type& team_member) { - - int p_id = team_member.league_rank() * team_size + team_member.team_rank(); - if (p_id >= number_of_unique_particles) return; - - Particle p = *unique_particles.at(p_id); - - write_particle(p, p_id, aosoa, box_l); - /*auto pos = p.pos(); - for (int d = 0; d < 3; ++d) { - double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; - aosoa.position(p_id, d) = wrapped; - } + Kokkos::parallel_for("AoSoA Write", policy_type(league_size, team_size), + [&unique_particles, &aosoa, &box_l, team_size, + number_of_unique_particles]( + const policy_type::member_type &team_member) { + int p_id = team_member.league_rank() * team_size + + team_member.team_rank(); + if (p_id >= number_of_unique_particles) + return; + + Particle p = *unique_particles.at(p_id); + + write_particle(p, p_id, aosoa, box_l); + /*auto pos = p.pos(); + for (int d = 0; d < 3; ++d) { + double wrapped = pos[d] - std::floor(pos[d] / + box_l[d]) * box_l[d]; aosoa.position(p_id, d) = + wrapped; + } - aosoa.charge(p_id) = p.q(); // charge - aosoa.id(p_id) = p.id(); // id - aosoa.type(p_id) = p.type(); // type - aosoa.ghost(p_id) = p.is_ghost(); // ghost*/ - }); + aosoa.charge(p_id) = p.q(); // charge + aosoa.id(p_id) = p.id(); // id + aosoa.type(p_id) = p.type(); // type + aosoa.ghost(p_id) = p.is_ghost(); // ghost*/ + }); Kokkos::fence(); Kokkos::View local_force( diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 7a4ebe5574f..32d41640423 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -299,11 +299,11 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // int jj = j; int jj = original_idx(j); int id_j = aosoa_id(jj); - /*if (id_i < id_j) { - if (aosoa_ghost(ii)) continue; - } else { - if (aosoa_ghost(jj)) continue; - }*/ + /*if (id_i < id_j) { + if (aosoa_ghost(ii)) continue; + } else { + if (aosoa_ghost(jj)) continue; + }*/ if (aosoa_ghost(ii) or aosoa_ghost(jj)) { if (((id_i < id_j) and aosoa_ghost(ii)) or ((id_i > id_j) and aosoa_ghost(jj))) { From 3a1d94cea42e1106137fb68461ed3e969a8cba88 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 3 Jul 2025 17:33:47 +0200 Subject: [PATCH 44/94] Added addNeighborNonAtomic --- src/core/short_range_cabana.hpp | 90 +++++++++++++------- src/core/verlet_list_loop.hpp | 141 ++++++++++++++++---------------- 2 files changed, 131 insertions(+), 100 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index eeebe38b8de..f5972f73bcb 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -196,43 +196,69 @@ void cabana_short_range( #endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); + particle_storage.resize(number_of_unique_particles); auto slice_position = Cabana::slice<0>(particle_storage); - auto slice_charge = Cabana::slice<1>(particle_storage); - auto slice_id = Cabana::slice<2>(particle_storage); - auto slice_type = Cabana::slice<3>(particle_storage); - auto slice_ghost = Cabana::slice<4>(particle_storage); + auto slice_charge = Cabana::slice<1>(particle_storage); + auto slice_id = Cabana::slice<2>(particle_storage); + auto slice_type = Cabana::slice<3>(particle_storage); + auto slice_ghost = Cabana::slice<4>(particle_storage); // particle properties are defined in aosoa_pack.hpp - particle_storage.resize(number_of_unique_particles); auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); + /* + using policy_type = Kokkos::RangePolicy; + Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), + [&unique_particles, &aosoa, &box_l]( + const int p_id) { + + Particle p = *unique_particles.at(p_id); + write_particle(p, p_id, aosoa, box_l); + }); + */ + using policy_type = Kokkos::TeamPolicy; - int team_size = 1; - int league_size = (unique_particles.size() + team_size - 1) / team_size; - Kokkos::parallel_for("AoSoA Write", policy_type(league_size, team_size), - [&unique_particles, &aosoa, &box_l, team_size, + int league_size = unique_particles.size(); + Kokkos::parallel_for("AoSoA Write", policy_type(league_size, Kokkos::AUTO), + [&unique_particles, &aosoa, &box_l, number_of_unique_particles]( const policy_type::member_type &team_member) { - int p_id = team_member.league_rank() * team_size + - team_member.team_rank(); + + int p_id = team_member.league_rank(); if (p_id >= number_of_unique_particles) return; Particle p = *unique_particles.at(p_id); write_particle(p, p_id, aosoa, box_l); - /*auto pos = p.pos(); - for (int d = 0; d < 3; ++d) { - double wrapped = pos[d] - std::floor(pos[d] / - box_l[d]) * box_l[d]; aosoa.position(p_id, d) = - wrapped; - } - - aosoa.charge(p_id) = p.q(); // charge - aosoa.id(p_id) = p.id(); // id - aosoa.type(p_id) = p.type(); // type - aosoa.ghost(p_id) = p.is_ghost(); // ghost*/ }); + + /*using policy_type = Cabana::SimdPolicy; + //int league_size = (particle_storage.size() + v_length - 1) / v_length; + //Kokkos::parallel_for("SIMD AoSoA Write", + Cabana::simd_parallel_for( + policy_type(0, particle_storage.size()), + [&unique_particles, &box_l, vector_length, + number_of_unique_particles, + &slice_position, &slice_charge, &slice_id, + &slice_type, &slice_ghost] (const int s, const int a) { + + int p_id = s * vector_length + a; + if (p_id >= number_of_unique_particles) return; + + Particle p = *unique_particles.at(p_id); + + auto pos = p.pos(); + for (int d = 0; d < 3; ++d) { + double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + slice_position.access(s, a, d) = wrapped; + } + + slice_charge.access(s, a) = p.q(); // charge + slice_id.access(s, a) = p.id(); // id + slice_type.access(s, a) = p.type(); // type + slice_ghost.access(s, a) = p.is_ghost(); // ghost + }, "SIMD AoSoA Write");*/ Kokkos::fence(); Kokkos::View local_force( @@ -457,7 +483,7 @@ void cabana_short_range( if (at_steepest_descent) { max_prefactor = 8; } else { - max_prefactor = 4; + max_prefactor = 5; } max_counts = static_cast( std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); @@ -469,8 +495,7 @@ void cabana_short_range( if (max_counts < threshold_num) { max_counts = std::min(threshold_num, number_of_unique_particles); } - // std::cout << "max_counts:" << max_counts << " " << max_cutoff << - // std::endl; + // std::cout << "max_counts:" << max_counts << " " << max_cutoff << std::endl; if (rebuild) { // Legacy Velert List /*verlet_list = ListType(aosoa.position, 0, aosoa.position.size(), max_counts, @@ -528,7 +553,8 @@ void cabana_short_range( #endif if (1) { verlet_list = create_verlet_list( - max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, + max_cutoff, max_counts, aosoa, + unique_particles, verlet_criterion, // sequential_particles, verlet_criterion, first_neighbor_kernel, cell_structure); /*using neighbor_list = Cabana::NeighborList; @@ -558,11 +584,13 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - calc Force"); #endif /* - Kokkos::parallel_for("ForceLoop", Kokkos::RangePolicy<>(0, - interaction_pairs.size()), KOKKOS_LAMBDA(int idx) { auto i = - interaction_pairs[idx].first; auto j = interaction_pairs[idx].second; - first_neighbor_kernel(i, j); - }); + Kokkos::parallel_for( + "ForceLoop", Kokkos::RangePolicy<>(0,interaction_pairs.size()), + KOKKOS_LAMBDA(int idx) { + auto i = interaction_pairs[idx].first; + auto j = interaction_pairs[idx].second; + first_neighbor_kernel(i, j); + }); */ // verlet_list.get_max_counts(); diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 32d41640423..f2bc991fb57 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -163,7 +163,8 @@ inline int set_interacting_pair_cell( // Interacting pair cell is registered in the list int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - if (cid_i <= cid_j) { + //if (cid_i <= cid_j) { + if (cid_i < cid_j) { if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { // std::size_t pcid = Kokkos::atomic_fetch_inc(&pair_cell_id()); // interacting_pair_cell(pcid, 0) = cid_i; @@ -270,7 +271,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, total_pair_cell = 14 * total_bins; } Kokkos::View interacting_pair_cell( - "interacting_pair_cell", total_pair_cell, 2); + "interacting_pair_cell", total_pair_cell - total_bins, 2); int empty_pair_number = set_interacting_pair_cell( total_bins, total_pair_cell, cell_num, delta_lebc, le_direction, le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); @@ -280,65 +281,58 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto aosoa_id = aosoa.id; auto aosoa_ghost = aosoa.ghost; - // This kernel used the loop for the pair of interacting cell - auto kernel = [&interacting_pair_cell, &bin_offset, &bin_size, &original_idx, + + // This kernel calculate within each cell + auto kernel_each = [&bin_offset, &bin_size, &original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, - &distance_function, &verlet_list, - &first_neighbor_kernel](const int pair_cell_i) { + &distance_function, &verlet_list, &first_neighbor_kernel](const int cid_i) { + auto thread_id = omp_get_thread_num(); + + int offset_i = bin_offset(cid_i); + int size_i = bin_size(cid_i); + + for (int i = offset_i; i < offset_i + size_i; ++i) { + // int ii = i; + int ii = original_idx(i); // get previous id + int id_i = aosoa_id(ii); + auto p1 = unique_particles.at(ii); + // auto p1 = cell_structure.get_local_particle(id_i); + for (int j = i + 1; j < offset_i + size_i; ++j) { + int jj = original_idx(j); + int id_j = aosoa_id(jj); + if (aosoa_ghost(ii) or aosoa_ghost(jj)) { + if (((id_i < id_j) and aosoa_ghost(ii)) or + ((id_i > id_j) and aosoa_ghost(jj))) { + continue; + } + } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { + continue; // reject both ghost + } + auto p2 = unique_particles.at(jj); + // auto p2 = cell_structure.get_local_particle(id_j); + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + //verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), std::max(ii, jj)); + verlet_list.addNeighborNonAtomic(thread_id, ii, jj); + first_neighbor_kernel(ii, jj); + } + } + } // i-loop + }; + + // This kernel used the loop for the pair of interacting cell + auto kernel_neighbor = [&interacting_pair_cell, &bin_offset, &bin_size, &original_idx, + &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, + &distance_function, &verlet_list, &first_neighbor_kernel](const int pair_cell_i) { int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); - auto verlet_kernel = - [&original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, - &verlet_criterion, &distance_function, &verlet_list, - &first_neighbor_kernel, thread_id](Particle *p1, int ii, int id_i, - int cell_offset, int cell_size) { - for (int j = cell_offset; j < cell_offset + cell_size; ++j) { - // int ii = cell_list.permutation(i); // debug - // int jj = j; - int jj = original_idx(j); - int id_j = aosoa_id(jj); - /*if (id_i < id_j) { - if (aosoa_ghost(ii)) continue; - } else { - if (aosoa_ghost(jj)) continue; - }*/ - if (aosoa_ghost(ii) or aosoa_ghost(jj)) { - if (((id_i < id_j) and aosoa_ghost(ii)) or - ((id_i > id_j) and aosoa_ghost(jj))) { - continue; - } - } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { - continue; // reject both ghost - } - auto p2 = unique_particles.at(jj); - // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(thread_id, ii, jj); - // verlet_list.addNeighborNonAtomic(thread_id, ii, jj); - /*std::cout << "*Ca* " - << ii << " " - << jj << " " - << aosoa_ghost(ii) << " " - << aosoa_ghost(jj) << " " - << id_i << " " - << id_j << "\n"; - << cid_i << " " - << cid_j << " " - << aosoa.position(ii, 0) << ", " - << aosoa.position(ii, 1) << ", " - << aosoa.position(ii, 2) << " " - << aosoa.position(jj, 0) << ", " - << aosoa.position(jj, 1) << ", " - << aosoa.position(jj, 2) << "\n";*/ - first_neighbor_kernel(ii, jj); - } - } // j-loop - }; + auto thread_id = omp_get_thread_num(); int offset_i = bin_offset(cid_i); int size_i = bin_size(cid_i); + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); for (int i = offset_i; i < offset_i + size_i; ++i) { // int ii = i; @@ -347,26 +341,35 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto p1 = unique_particles.at(ii); // auto p1 = cell_structure.get_local_particle(id_i); - if (cid_i == cid_j) { - verlet_kernel(p1, ii, id_i, i + 1, - size_i + offset_i - i - 1); // j-loop - // verlet_kernel(p1, i, id_i, i + 1, - // size_i + offset_i - i - 1); // j-loop - } else { - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - verlet_kernel(p1, ii, id_i, offset_j, size_j); // j-loop - // verlet_kernel(p1, i, id_i, offset_j, size_j); // j-loop - } - } // i-loop + for (int j = offset_j; j < offset_j + size_j; ++j) { + int jj = original_idx(j); + int id_j = aosoa_id(jj); + if (aosoa_ghost(ii) or aosoa_ghost(jj)) { + if (((id_i < id_j) and aosoa_ghost(ii)) or + ((id_i > id_j) and aosoa_ghost(jj))) { + continue; + } + } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { + continue; // reject both ghost + } + auto p2 = unique_particles.at(jj); + // auto p2 = cell_structure.get_local_particle(id_j); + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(thread_id, ii, jj); + first_neighbor_kernel(ii, jj); + } + } // i-loop + } }; - Kokkos::RangePolicy policy(0, total_pair_cell - - empty_pair_number); - Kokkos::parallel_for("calc_by_cell_list", policy, kernel); + Kokkos::RangePolicy policy_each(0, total_bins); + Kokkos::parallel_for("calc_by_cell_list_each", policy_each, kernel_each); Kokkos::fence(); - // verlet_list.reduction(); + Kokkos::RangePolicy policy_neighbor(0, total_pair_cell -total_bins - + empty_pair_number); + Kokkos::parallel_for("calc_by_cell_list_beighbor", policy_neighbor, kernel_neighbor); + Kokkos::fence(); return verlet_list; } From 74d65fa9f4fdefd0f791d90ee254d6998722174d Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 3 Jul 2025 17:34:48 +0200 Subject: [PATCH 45/94] Added new function in custom_verlet_list.hpp --- src/core/custom_verlet_list.hpp | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 929236f6dfe..523ba6fb584 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -66,19 +66,6 @@ class CustomVerletList for (int tid = 0; tid < thread_number; ++tid) { max_thread(tid) = 1; } - /* - counts_thread = Kokkos::View( - "num_neighbors", thread_number, num_particles); - neighbors_thread = Kokkos::View( - Kokkos::ViewAllocateWithoutInitializing("neighbors"), thread_number, - num_particles, max_neigh); - Kokkos::parallel_for("initialize counts_thread", num_particles, - [=, this](const int &i) { - for (int tid = 0; tid < thread_number; ++tid) { - counts_thread(tid, i) = 0; - } - }); - */ } // Method to dynamically expand the size of max_neighbors @@ -123,13 +110,18 @@ class CustomVerletList // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION - void addNeighborNonAtomic(const int tid, const int pid, const int nid) { - neighbors_thread(tid, pid, counts_thread(tid, pid)) = nid; - counts_thread(tid, pid) += 1; - if (counts_thread(tid, pid) >= neighbors.extent(1)) { + void addNeighborNonAtomic(const int tid, int pid, int nid) { + if (counts(pid) + 1 > max_thread(tid)) + std::swap(pid, nid); + std::size_t count = counts(pid); + counts(pid) += 1; + if (count >= neighbors.extent(1)) { throw std::runtime_error( "Number of count in one thread is larger than VerletList size."); } + neighbors(pid, count) = nid; + if (counts(pid) > max_thread(tid)) + max_thread(tid) = counts(pid); } // Reduction of counts and neighbor in all threads From 78b11ea903df5f6acaef5f48615a245e3da29159 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 3 Jul 2025 17:36:36 +0200 Subject: [PATCH 46/94] Formatting --- src/core/short_range_cabana.hpp | 68 ++++++++-------- src/core/verlet_list_loop.hpp | 133 ++++++++++++++++---------------- 2 files changed, 102 insertions(+), 99 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index f5972f73bcb..e4a94ae0544 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -198,10 +198,10 @@ void cabana_short_range( "particles", number_of_unique_particles); particle_storage.resize(number_of_unique_particles); auto slice_position = Cabana::slice<0>(particle_storage); - auto slice_charge = Cabana::slice<1>(particle_storage); - auto slice_id = Cabana::slice<2>(particle_storage); - auto slice_type = Cabana::slice<3>(particle_storage); - auto slice_ghost = Cabana::slice<4>(particle_storage); + auto slice_charge = Cabana::slice<1>(particle_storage); + auto slice_id = Cabana::slice<2>(particle_storage); + auto slice_type = Cabana::slice<3>(particle_storage); + auto slice_ghost = Cabana::slice<4>(particle_storage); // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); @@ -216,48 +216,48 @@ void cabana_short_range( write_particle(p, p_id, aosoa, box_l); }); */ - + using policy_type = Kokkos::TeamPolicy; int league_size = unique_particles.size(); - Kokkos::parallel_for("AoSoA Write", policy_type(league_size, Kokkos::AUTO), - [&unique_particles, &aosoa, &box_l, - number_of_unique_particles]( - const policy_type::member_type &team_member) { + Kokkos::parallel_for( + "AoSoA Write", policy_type(league_size, Kokkos::AUTO), + [&unique_particles, &aosoa, &box_l, number_of_unique_particles]( + const policy_type::member_type &team_member) { + int p_id = team_member.league_rank(); + if (p_id >= number_of_unique_particles) + return; - int p_id = team_member.league_rank(); - if (p_id >= number_of_unique_particles) - return; + Particle p = *unique_particles.at(p_id); - Particle p = *unique_particles.at(p_id); + write_particle(p, p_id, aosoa, box_l); + }); - write_particle(p, p_id, aosoa, box_l); - }); - /*using policy_type = Cabana::SimdPolicy; //int league_size = (particle_storage.size() + v_length - 1) / v_length; //Kokkos::parallel_for("SIMD AoSoA Write", Cabana::simd_parallel_for( - policy_type(0, particle_storage.size()), + policy_type(0, particle_storage.size()), [&unique_particles, &box_l, vector_length, number_of_unique_particles, - &slice_position, &slice_charge, &slice_id, - &slice_type, &slice_ghost] (const int s, const int a) { + &slice_position, &slice_charge, &slice_id, + &slice_type, &slice_ghost] (const int s, const int a) + { int p_id = s * vector_length + a; if (p_id >= number_of_unique_particles) return; Particle p = *unique_particles.at(p_id); - auto pos = p.pos(); - for (int d = 0; d < 3; ++d) { - double wrapped = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; - slice_position.access(s, a, d) = wrapped; - } + auto pos = p.pos(); + for (int d = 0; d < 3; ++d) { + double wrapped = pos[d] - std::floor(pos[d] / + box_l[d]) * box_l[d]; slice_position.access(s, a, d) = wrapped; + } - slice_charge.access(s, a) = p.q(); // charge - slice_id.access(s, a) = p.id(); // id - slice_type.access(s, a) = p.type(); // type - slice_ghost.access(s, a) = p.is_ghost(); // ghost + slice_charge.access(s, a) = p.q(); // charge + slice_id.access(s, a) = p.id(); // id + slice_type.access(s, a) = p.type(); // type + slice_ghost.access(s, a) = p.is_ghost(); // ghost }, "SIMD AoSoA Write");*/ Kokkos::fence(); @@ -495,7 +495,8 @@ void cabana_short_range( if (max_counts < threshold_num) { max_counts = std::min(threshold_num, number_of_unique_particles); } - // std::cout << "max_counts:" << max_counts << " " << max_cutoff << std::endl; + // std::cout << "max_counts:" << max_counts << " " << max_cutoff << + // std::endl; if (rebuild) { // Legacy Velert List /*verlet_list = ListType(aosoa.position, 0, aosoa.position.size(), max_counts, @@ -553,8 +554,7 @@ void cabana_short_range( #endif if (1) { verlet_list = create_verlet_list( - max_cutoff, max_counts, aosoa, - unique_particles, verlet_criterion, + max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, // sequential_particles, verlet_criterion, first_neighbor_kernel, cell_structure); /*using neighbor_list = Cabana::NeighborList; @@ -586,9 +586,9 @@ void cabana_short_range( /* Kokkos::parallel_for( "ForceLoop", Kokkos::RangePolicy<>(0,interaction_pairs.size()), - KOKKOS_LAMBDA(int idx) { - auto i = interaction_pairs[idx].first; - auto j = interaction_pairs[idx].second; + KOKKOS_LAMBDA(int idx) { + auto i = interaction_pairs[idx].first; + auto j = interaction_pairs[idx].second; first_neighbor_kernel(i, j); }); */ diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index f2bc991fb57..5e50bb9d171 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -163,7 +163,7 @@ inline int set_interacting_pair_cell( // Interacting pair cell is registered in the list int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - //if (cid_i <= cid_j) { + // if (cid_i <= cid_j) { if (cid_i < cid_j) { if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { // std::size_t pcid = Kokkos::atomic_fetch_inc(&pair_cell_id()); @@ -283,10 +283,10 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto aosoa_ghost = aosoa.ghost; // This kernel calculate within each cell - auto kernel_each = [&bin_offset, &bin_size, &original_idx, - &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, - &distance_function, &verlet_list, &first_neighbor_kernel](const int cid_i) { - + auto kernel_each = [&bin_offset, &bin_size, &original_idx, &aosoa_id, + &aosoa_ghost, &unique_particles, &verlet_criterion, + &distance_function, &verlet_list, + &first_neighbor_kernel](const int cid_i) { auto thread_id = omp_get_thread_num(); int offset_i = bin_offset(cid_i); @@ -299,76 +299,79 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto p1 = unique_particles.at(ii); // auto p1 = cell_structure.get_local_particle(id_i); for (int j = i + 1; j < offset_i + size_i; ++j) { - int jj = original_idx(j); - int id_j = aosoa_id(jj); - if (aosoa_ghost(ii) or aosoa_ghost(jj)) { - if (((id_i < id_j) and aosoa_ghost(ii)) or - ((id_i > id_j) and aosoa_ghost(jj))) { - continue; - } - } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { - continue; // reject both ghost - } - auto p2 = unique_particles.at(jj); - // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - //verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), std::max(ii, jj)); - verlet_list.addNeighborNonAtomic(thread_id, ii, jj); - first_neighbor_kernel(ii, jj); - } + int jj = original_idx(j); + int id_j = aosoa_id(jj); + if (aosoa_ghost(ii) or aosoa_ghost(jj)) { + if (((id_i < id_j) and aosoa_ghost(ii)) or + ((id_i > id_j) and aosoa_ghost(jj))) { + continue; + } + } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { + continue; // reject both ghost + } + auto p2 = unique_particles.at(jj); + // auto p2 = cell_structure.get_local_particle(id_j); + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + // verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), + // std::max(ii, jj)); + verlet_list.addNeighborNonAtomic(thread_id, ii, jj); + first_neighbor_kernel(ii, jj); + } } } // i-loop }; // This kernel used the loop for the pair of interacting cell - auto kernel_neighbor = [&interacting_pair_cell, &bin_offset, &bin_size, &original_idx, - &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, - &distance_function, &verlet_list, &first_neighbor_kernel](const int pair_cell_i) { - int cid_i = interacting_pair_cell(pair_cell_i, 0); - int cid_j = interacting_pair_cell(pair_cell_i, 1); - - auto thread_id = omp_get_thread_num(); - - int offset_i = bin_offset(cid_i); - int size_i = bin_size(cid_i); - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - - for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; - int ii = original_idx(i); // get previous id - int id_i = aosoa_id(ii); - auto p1 = unique_particles.at(ii); - // auto p1 = cell_structure.get_local_particle(id_i); - - for (int j = offset_j; j < offset_j + size_j; ++j) { - int jj = original_idx(j); - int id_j = aosoa_id(jj); - if (aosoa_ghost(ii) or aosoa_ghost(jj)) { - if (((id_i < id_j) and aosoa_ghost(ii)) or - ((id_i > id_j) and aosoa_ghost(jj))) { - continue; - } - } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { - continue; // reject both ghost - } - auto p2 = unique_particles.at(jj); - // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - verlet_list.addNeighbor(thread_id, ii, jj); - first_neighbor_kernel(ii, jj); - } - } // i-loop - } - }; + auto kernel_neighbor = + [&interacting_pair_cell, &bin_offset, &bin_size, &original_idx, &aosoa_id, + &aosoa_ghost, &unique_particles, &verlet_criterion, &distance_function, + &verlet_list, &first_neighbor_kernel](const int pair_cell_i) { + int cid_i = interacting_pair_cell(pair_cell_i, 0); + int cid_j = interacting_pair_cell(pair_cell_i, 1); + + auto thread_id = omp_get_thread_num(); + + int offset_i = bin_offset(cid_i); + int size_i = bin_size(cid_i); + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); + + for (int i = offset_i; i < offset_i + size_i; ++i) { + // int ii = i; + int ii = original_idx(i); // get previous id + int id_i = aosoa_id(ii); + auto p1 = unique_particles.at(ii); + // auto p1 = cell_structure.get_local_particle(id_i); + + for (int j = offset_j; j < offset_j + size_j; ++j) { + int jj = original_idx(j); + int id_j = aosoa_id(jj); + if (aosoa_ghost(ii) or aosoa_ghost(jj)) { + if (((id_i < id_j) and aosoa_ghost(ii)) or + ((id_i > id_j) and aosoa_ghost(jj))) { + continue; + } + } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { + continue; // reject both ghost + } + auto p2 = unique_particles.at(jj); + // auto p2 = cell_structure.get_local_particle(id_j); + if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + verlet_list.addNeighbor(thread_id, ii, jj); + first_neighbor_kernel(ii, jj); + } + } // i-loop + } + }; Kokkos::RangePolicy policy_each(0, total_bins); Kokkos::parallel_for("calc_by_cell_list_each", policy_each, kernel_each); Kokkos::fence(); - Kokkos::RangePolicy policy_neighbor(0, total_pair_cell -total_bins - - empty_pair_number); - Kokkos::parallel_for("calc_by_cell_list_beighbor", policy_neighbor, kernel_neighbor); + Kokkos::RangePolicy policy_neighbor( + 0, total_pair_cell - total_bins - empty_pair_number); + Kokkos::parallel_for("calc_by_cell_list_beighbor", policy_neighbor, + kernel_neighbor); Kokkos::fence(); return verlet_list; From 3dc6b795793de226c7c1f1a10d0ee2fd16cdef1c Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 4 Jul 2025 20:20:10 +0200 Subject: [PATCH 47/94] Modifed addNeighbor --- src/core/aosoa_pack.hpp | 2 +- src/core/custom_verlet_list.hpp | 88 +++++++----------------- src/core/short_range_cabana.hpp | 69 +++---------------- src/core/verlet_list_loop.hpp | 31 ++++----- testsuite/python/integrator_npt_stats.py | 4 +- 5 files changed, 51 insertions(+), 143 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index d745bb82492..c710dfe9b12 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -23,7 +23,7 @@ #include -const int vector_length = 1; +const int vector_length = 32; using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 523ba6fb584..06e8702cdee 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -46,8 +46,6 @@ class CustomVerletList private: Kokkos::View max_thread; - Kokkos::View counts_thread; - Kokkos::View neighbors_thread; public: Kokkos::View counts; @@ -68,83 +66,49 @@ class CustomVerletList } } - // Method to dynamically expand the size of max_neighbors - // This function may be vaiolated Kokkos's rule. - // Kokkos::View should not be created in Kokkos::parallel. - // However, this function is called from addNeighbor used - // in the Kokkos::parallel. - KOKKOS_INLINE_FUNCTION - void expandMaxNeighbors(const std::size_t new_max_neigh) { - // Create a new view with the larger size - Kokkos::View new_neighbors( - Kokkos::ViewAllocateWithoutInitializing("neighbors"), - neighbors.extent(0), new_max_neigh); - - // Copy existing data to the new view - Kokkos::parallel_for("copy_neighbors", neighbors.extent(0), - [=, this](const int i) { - for (std::size_t j = 0; j < counts(i); ++j) { - new_neighbors(i, j) = neighbors(i, j); - } - }); - - // Replace the old view with the new view - neighbors = new_neighbors; - } - // Method to add a neighbor KOKKOS_INLINE_FUNCTION void addNeighbor(const int tid, int pid, int nid) { - if (counts(pid) + 1 > max_thread(tid)) - std::swap(pid, nid); - std::size_t count = Kokkos::atomic_fetch_add(&counts(pid), 1); + std::size_t count = counts(pid); + std::size_t max_t = max_thread(tid); + + if (count + 1 > max_t) { + int tmp = pid; + pid = nid; + nid = tmp; + } + count = Kokkos::atomic_fetch_add(&counts(pid), 1); if (count >= neighbors.extent(1)) { - // expandMaxNeighbors(neighbors.extent(1) * 2); throw std::runtime_error( - "Number of count is larger than VerletList size."); + "Number of count in one thread is larger than VerletList size."); } neighbors(pid, count) = nid; - if (counts(pid) > max_thread(tid)) - max_thread(tid) = counts(pid); + std::size_t new_count = count + 1; + if (new_count > max_t) + max_thread(tid) = new_count; } // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION void addNeighborNonAtomic(const int tid, int pid, int nid) { - if (counts(pid) + 1 > max_thread(tid)) - std::swap(pid, nid); std::size_t count = counts(pid); - counts(pid) += 1; + std::size_t max_t = max_thread(tid); + + if (count + 1 > max_t) { + int tmp = pid; + pid = nid; + nid = tmp; + count = counts(pid); + } if (count >= neighbors.extent(1)) { + // expandMaxNeighbors(neighbors.extent(1) * 2); throw std::runtime_error( - "Number of count in one thread is larger than VerletList size."); + "Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; - if (counts(pid) > max_thread(tid)) - max_thread(tid) = counts(pid); - } - - // Reduction of counts and neighbor in all threads - KOKKOS_INLINE_FUNCTION - void reduction() { - // Kokkos::RangePolicy policy(0, counts.extent(0)); - int thread_number = counts_thread.extent(0); - Kokkos::parallel_for( - "reduction_neighbor", counts.extent(0), [&](const int pid) { - counts(pid) = 0; - for (int tid = 0; tid < thread_number; ++tid) { - std::size_t offset = counts(pid); - counts(pid) += counts_thread(tid, pid); - if (counts(pid) >= neighbors.extent(1)) { - throw std::runtime_error( - "Number of count is larger than VerletList size."); - } - for (int cid = 0; cid < counts_thread(tid, pid); ++cid) { - neighbors(pid, offset + cid) = neighbors_thread(tid, pid, cid); - } - } - }); - Kokkos::fence(); + counts(pid) += 1; + if (count + 1 > max_thread(tid)) + max_thread(tid) = count + 1; } // Find max counts diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index e4a94ae0544..9c873082463 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -50,7 +50,7 @@ inline double wrap(double x, double L) { } inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, - Utils::Vector3d &box_l) { + Utils::Vector3d const &box_l) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); @@ -194,71 +194,18 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Allocation"); #endif - Cabana::AoSoA particle_storage( - "particles", number_of_unique_particles); + Cabana::AoSoA + particle_storage("particles", number_of_unique_particles); particle_storage.resize(number_of_unique_particles); - auto slice_position = Cabana::slice<0>(particle_storage); - auto slice_charge = Cabana::slice<1>(particle_storage); - auto slice_id = Cabana::slice<2>(particle_storage); - auto slice_type = Cabana::slice<3>(particle_storage); - auto slice_ghost = Cabana::slice<4>(particle_storage); // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); - /* + using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), - [&unique_particles, &aosoa, &box_l]( - const int p_id) { - - Particle p = *unique_particles.at(p_id); - - write_particle(p, p_id, aosoa, box_l); + [&unique_particles, &aosoa, &box_l](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); }); - */ - - using policy_type = Kokkos::TeamPolicy; - int league_size = unique_particles.size(); - Kokkos::parallel_for( - "AoSoA Write", policy_type(league_size, Kokkos::AUTO), - [&unique_particles, &aosoa, &box_l, number_of_unique_particles]( - const policy_type::member_type &team_member) { - int p_id = team_member.league_rank(); - if (p_id >= number_of_unique_particles) - return; - - Particle p = *unique_particles.at(p_id); - - write_particle(p, p_id, aosoa, box_l); - }); - - /*using policy_type = Cabana::SimdPolicy; - //int league_size = (particle_storage.size() + v_length - 1) / v_length; - //Kokkos::parallel_for("SIMD AoSoA Write", - Cabana::simd_parallel_for( - policy_type(0, particle_storage.size()), - [&unique_particles, &box_l, vector_length, - number_of_unique_particles, - &slice_position, &slice_charge, &slice_id, - &slice_type, &slice_ghost] (const int s, const int a) - { - - int p_id = s * vector_length + a; - if (p_id >= number_of_unique_particles) return; - - Particle p = *unique_particles.at(p_id); - - auto pos = p.pos(); - for (int d = 0; d < 3; ++d) { - double wrapped = pos[d] - std::floor(pos[d] / - box_l[d]) * box_l[d]; slice_position.access(s, a, d) = wrapped; - } - - slice_charge.access(s, a) = p.q(); // charge - slice_id.access(s, a) = p.id(); // id - slice_type.access(s, a) = p.type(); // type - slice_ghost.access(s, a) = p.is_ghost(); // ghost - }, "SIMD AoSoA Write");*/ Kokkos::fence(); Kokkos::View local_force( @@ -283,7 +230,7 @@ void cabana_short_range( [[maybe_unused]] const BondedInteractionsMap &bonded_ias; const InteractionsNonBonded &nonbonded_ias; const BoxGeometry &box_geo; - AoSoA_pack aosoa; + const AoSoA_pack aosoa; Kokkos::View local_force; #ifdef ROTATION Kokkos::View local_torque; @@ -317,7 +264,7 @@ void cabana_short_range( #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, AoSoA_pack &aosoa_, + const BoxGeometry &box_geo_, const AoSoA_pack &aosoa_, Kokkos::View local_force_, #ifdef ROTATION Kokkos::View local_torque_, diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 5e50bb9d171..b81dda1fd92 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -97,10 +97,6 @@ inline int set_interacting_pair_cell( int empty_pair_number = 0; int pair_cell_id = 0; - // Kokkos::View empty_pair_number("empty_pair_number"); - // Kokkos::View pair_cell_id("pair_cell_id"); - // Kokkos::deep_copy(empty_pair_number, 0); - // Kokkos::deep_copy(pair_cell_id, 0); for (int cid_i = 0; cid_i < total_bins; ++cid_i) { // Kokkos::parallel_for("set_interacting_pair_cell", total_bins, @@ -195,7 +191,6 @@ template ListType create_verlet_list(double const max_cutoff, int const max_counts, AoSoA_pack &aosoa, std::vector &unique_particles, - // std::vector &unique_particles, VerletCriterion const &verlet_criterion, Kernel &first_neighbor_kernel, CellStructure &cell_structure) { @@ -296,9 +291,10 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // int ii = i; int ii = original_idx(i); // get previous id int id_i = aosoa_id(ii); - auto p1 = unique_particles.at(ii); + //auto p1 = unique_particles.at(ii); // auto p1 = cell_structure.get_local_particle(id_i); for (int j = i + 1; j < offset_i + size_i; ++j) { + // int jj = j; int jj = original_idx(j); int id_j = aosoa_id(jj); if (aosoa_ghost(ii) or aosoa_ghost(jj)) { @@ -306,14 +302,13 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, ((id_i > id_j) and aosoa_ghost(jj))) { continue; } - } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { - continue; // reject both ghost } - auto p2 = unique_particles.at(jj); + //auto p2 = unique_particles.at(jj); // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { - // verlet_list.addNeighborNonAtomic(thread_id, std::min(ii, jj), - // std::max(ii, jj)); + if (verlet_criterion(*unique_particles.at(ii), + *unique_particles.at(jj), + distance_function(*unique_particles.at(ii), + *unique_particles.at(jj)))) { verlet_list.addNeighborNonAtomic(thread_id, ii, jj); first_neighbor_kernel(ii, jj); } @@ -340,10 +335,11 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // int ii = i; int ii = original_idx(i); // get previous id int id_i = aosoa_id(ii); - auto p1 = unique_particles.at(ii); + // auto p1 = unique_particles.at(ii); // auto p1 = cell_structure.get_local_particle(id_i); for (int j = offset_j; j < offset_j + size_j; ++j) { + // int jj = j; int jj = original_idx(j); int id_j = aosoa_id(jj); if (aosoa_ghost(ii) or aosoa_ghost(jj)) { @@ -351,12 +347,13 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, ((id_i > id_j) and aosoa_ghost(jj))) { continue; } - } else if (aosoa_ghost(ii) and aosoa_ghost(jj)) { - continue; // reject both ghost } - auto p2 = unique_particles.at(jj); + // auto p2 = unique_particles.at(jj); // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *p2, distance_function(*p1, *p2))) { + if (verlet_criterion(*unique_particles.at(ii), + *unique_particles.at(jj), + distance_function(*unique_particles.at(ii), + *unique_particles.at(jj)))) { verlet_list.addNeighbor(thread_id, ii, jj); first_neighbor_kernel(ii, jj); } diff --git a/testsuite/python/integrator_npt_stats.py b/testsuite/python/integrator_npt_stats.py index 00a1c8eb51b..c25044e6d6d 100644 --- a/testsuite/python/integrator_npt_stats.py +++ b/testsuite/python/integrator_npt_stats.py @@ -113,8 +113,8 @@ def test_compressibility_and_pressure(self): self.assertAlmostEqual(avp, p_ext, delta=0.02) self.assertAlmostEqual(compressibility, 0.5, delta=0.05) np.testing.assert_allclose(avp_sim_vir, avp_inst_vir, atol=1e-10) - self.assertAlmostEqual(avpV_sim, 100., delta=1.5) - self.assertAlmostEqual(avpV_inst, 100., delta=1.5) + self.assertAlmostEqual(avpV_sim, 100., delta=1.0) + self.assertAlmostEqual(avpV_inst, 100., delta=1.0) def test_negative_volume(self): """Test for NpT with bad parameters.""" From 73ef5d7a6fb69a34968f92fbaa82e3dd48537f3f Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 4 Jul 2025 20:21:18 +0200 Subject: [PATCH 48/94] Formatting --- src/core/short_range_cabana.hpp | 9 +-- src/core/verlet_list_loop.hpp | 99 ++++++++++++++++----------------- 2 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 9c873082463..b1b6d47a5ca 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -194,17 +194,18 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Allocation"); #endif - Cabana::AoSoA - particle_storage("particles", number_of_unique_particles); + Cabana::AoSoA particle_storage( + "particles", number_of_unique_particles); particle_storage.resize(number_of_unique_particles); // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); - + using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), [&unique_particles, &aosoa, &box_l](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); + write_particle(*unique_particles.at(p_id), p_id, + aosoa, box_l); }); Kokkos::fence(); diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index b81dda1fd92..d9f69385d41 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -291,8 +291,8 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // int ii = i; int ii = original_idx(i); // get previous id int id_i = aosoa_id(ii); - //auto p1 = unique_particles.at(ii); - // auto p1 = cell_structure.get_local_particle(id_i); + // auto p1 = unique_particles.at(ii); + // auto p1 = cell_structure.get_local_particle(id_i); for (int j = i + 1; j < offset_i + size_i; ++j) { // int jj = j; int jj = original_idx(j); @@ -303,12 +303,11 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, continue; } } - //auto p2 = unique_particles.at(jj); - // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*unique_particles.at(ii), - *unique_particles.at(jj), - distance_function(*unique_particles.at(ii), - *unique_particles.at(jj)))) { + // auto p2 = unique_particles.at(jj); + // auto p2 = cell_structure.get_local_particle(id_j); + if (verlet_criterion(*unique_particles.at(ii), *unique_particles.at(jj), + distance_function(*unique_particles.at(ii), + *unique_particles.at(jj)))) { verlet_list.addNeighborNonAtomic(thread_id, ii, jj); first_neighbor_kernel(ii, jj); } @@ -317,49 +316,49 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, }; // This kernel used the loop for the pair of interacting cell - auto kernel_neighbor = - [&interacting_pair_cell, &bin_offset, &bin_size, &original_idx, &aosoa_id, - &aosoa_ghost, &unique_particles, &verlet_criterion, &distance_function, - &verlet_list, &first_neighbor_kernel](const int pair_cell_i) { - int cid_i = interacting_pair_cell(pair_cell_i, 0); - int cid_j = interacting_pair_cell(pair_cell_i, 1); - - auto thread_id = omp_get_thread_num(); - - int offset_i = bin_offset(cid_i); - int size_i = bin_size(cid_i); - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - - for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; - int ii = original_idx(i); // get previous id - int id_i = aosoa_id(ii); - // auto p1 = unique_particles.at(ii); - // auto p1 = cell_structure.get_local_particle(id_i); - - for (int j = offset_j; j < offset_j + size_j; ++j) { - // int jj = j; - int jj = original_idx(j); - int id_j = aosoa_id(jj); - if (aosoa_ghost(ii) or aosoa_ghost(jj)) { - if (((id_i < id_j) and aosoa_ghost(ii)) or - ((id_i > id_j) and aosoa_ghost(jj))) { - continue; - } - } - // auto p2 = unique_particles.at(jj); - // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*unique_particles.at(ii), - *unique_particles.at(jj), - distance_function(*unique_particles.at(ii), - *unique_particles.at(jj)))) { - verlet_list.addNeighbor(thread_id, ii, jj); - first_neighbor_kernel(ii, jj); - } - } // i-loop + auto kernel_neighbor = [&interacting_pair_cell, &bin_offset, &bin_size, + &original_idx, &aosoa_id, &aosoa_ghost, + &unique_particles, &verlet_criterion, + &distance_function, &verlet_list, + &first_neighbor_kernel](const int pair_cell_i) { + int cid_i = interacting_pair_cell(pair_cell_i, 0); + int cid_j = interacting_pair_cell(pair_cell_i, 1); + + auto thread_id = omp_get_thread_num(); + + int offset_i = bin_offset(cid_i); + int size_i = bin_size(cid_i); + int offset_j = bin_offset(cid_j); + int size_j = bin_size(cid_j); + + for (int i = offset_i; i < offset_i + size_i; ++i) { + // int ii = i; + int ii = original_idx(i); // get previous id + int id_i = aosoa_id(ii); + // auto p1 = unique_particles.at(ii); + // auto p1 = cell_structure.get_local_particle(id_i); + + for (int j = offset_j; j < offset_j + size_j; ++j) { + // int jj = j; + int jj = original_idx(j); + int id_j = aosoa_id(jj); + if (aosoa_ghost(ii) or aosoa_ghost(jj)) { + if (((id_i < id_j) and aosoa_ghost(ii)) or + ((id_i > id_j) and aosoa_ghost(jj))) { + continue; + } + } + // auto p2 = unique_particles.at(jj); + // auto p2 = cell_structure.get_local_particle(id_j); + if (verlet_criterion(*unique_particles.at(ii), *unique_particles.at(jj), + distance_function(*unique_particles.at(ii), + *unique_particles.at(jj)))) { + verlet_list.addNeighbor(thread_id, ii, jj); + first_neighbor_kernel(ii, jj); } - }; + } // i-loop + } + }; Kokkos::RangePolicy policy_each(0, total_bins); Kokkos::parallel_for("calc_by_cell_list_each", policy_each, kernel_each); From dca72ed2d27060ad9fdedebe1757187dd1042b02 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 7 Jul 2025 12:42:41 +0200 Subject: [PATCH 49/94] Modified verlet_list_loop --- src/core/short_range_cabana.hpp | 18 +++++++++--------- src/core/verlet_list_loop.hpp | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index b1b6d47a5ca..9ec856d3a8a 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -194,6 +194,15 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Allocation"); #endif + Kokkos::View local_force( + "local_force", num_threads, number_of_unique_particles, 3); + + Kokkos::View local_torque( + "local_torque", num_threads, number_of_unique_particles, 3); + + Kokkos::View local_virial("local_virial", + num_threads, 3); + Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); particle_storage.resize(number_of_unique_particles); @@ -209,15 +218,6 @@ void cabana_short_range( }); Kokkos::fence(); - Kokkos::View local_force( - "local_force", num_threads, number_of_unique_particles, 3); - - Kokkos::View local_torque( - "local_torque", num_threads, number_of_unique_particles, 3); - - Kokkos::View local_virial("local_virial", - num_threads, 3); - #ifdef CALIPER CALI_MARK_END("Cabana - Allocation"); #endif diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index d9f69385d41..5d8aa082197 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -291,7 +291,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // int ii = i; int ii = original_idx(i); // get previous id int id_i = aosoa_id(ii); - // auto p1 = unique_particles.at(ii); + auto p1 = unique_particles.at(ii); // auto p1 = cell_structure.get_local_particle(id_i); for (int j = i + 1; j < offset_i + size_i; ++j) { // int jj = j; @@ -305,8 +305,8 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, } // auto p2 = unique_particles.at(jj); // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*unique_particles.at(ii), *unique_particles.at(jj), - distance_function(*unique_particles.at(ii), + if (verlet_criterion(*p1, *unique_particles.at(jj), + distance_function(*p1, *unique_particles.at(jj)))) { verlet_list.addNeighborNonAtomic(thread_id, ii, jj); first_neighbor_kernel(ii, jj); @@ -335,7 +335,7 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // int ii = i; int ii = original_idx(i); // get previous id int id_i = aosoa_id(ii); - // auto p1 = unique_particles.at(ii); + auto p1 = unique_particles.at(ii); // auto p1 = cell_structure.get_local_particle(id_i); for (int j = offset_j; j < offset_j + size_j; ++j) { @@ -350,8 +350,8 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, } // auto p2 = unique_particles.at(jj); // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*unique_particles.at(ii), *unique_particles.at(jj), - distance_function(*unique_particles.at(ii), + if (verlet_criterion(*p1, *unique_particles.at(jj), + distance_function(*p1, *unique_particles.at(jj)))) { verlet_list.addNeighbor(thread_id, ii, jj); first_neighbor_kernel(ii, jj); From 1763d69403b1132fb9029c678ceedb6d2d8736a3 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 9 Jul 2025 19:42:18 +0200 Subject: [PATCH 50/94] Experiment for computational rate --- src/core/aosoa_pack.hpp | 4 +- src/core/cabana_data.hpp | 12 +- src/core/cell_system/CellStructure.hpp | 22 +++- src/core/custom_verlet_list.hpp | 41 +++---- src/core/short_range_cabana.hpp | 148 ++++++++++++++++++------- src/core/verlet_list_loop.hpp | 33 +++--- 6 files changed, 169 insertions(+), 91 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index c710dfe9b12..5987f6787b8 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -23,8 +23,8 @@ #include -const int vector_length = 32; -using data_types = Cabana::MemberTypes; +const int vector_length = 1; +using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using AoSoA_type = Cabana::AoSoA; diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index b22549eb7b9..537ca6bc5c8 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -37,20 +37,20 @@ class CabanaData { private: ListType verlet_list; std::vector unique_particles; - // int max_id; + int max_id; public: CabanaData() = default; CabanaData(ListType verlet_list, std::vector unique_particles) : verlet_list(verlet_list), unique_particles(unique_particles) {} - // CabanaData(ListType verlet_list, std::vector unique_particles, - // int max_id) - // : verlet_list(verlet_list), unique_particles(unique_particles), - // max_id(max_id) {} + CabanaData(ListType verlet_list, std::vector unique_particles, + int max_id) + : verlet_list(verlet_list), unique_particles(unique_particles), + max_id(max_id) {} ListType get_verlet_list() const { return verlet_list; } int get_index() const { return unique_particles.size(); } - // int get_max_id() const { return max_id; } + int get_max_id() const { return max_id; } std::vector get_unique_particles() const { return unique_particles; } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 7db44629d0d..2cff65e54da 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -57,6 +57,10 @@ #include #include +#ifdef CALIPER +#include +#endif + // forward declaration to not have to import cabana #ifdef SHARED_MEMORY_PARALLELISM class CabanaData; @@ -699,14 +703,12 @@ struct CellStructure : public System::Leaf { link_cell([&](Particle &p1, Particle &p2, Distance const &d) { if (verlet_criterion(p1, p2, d)) { m_verlet_list.emplace_back(&p1, &p2); + kernel(p1, p2); } }); m_rebuild_verlet_list = false; + m_rebuild_cabana_verlet_list = false; } - for (auto const &pair : m_verlet_list) { - kernel(*pair.first, *pair.second); - } - m_rebuild_cabana_verlet_list = false; } #endif @@ -723,6 +725,9 @@ struct CellStructure : public System::Leaf { * the pair kernel, and the verlet list is rebuilt as * we go. */ if (m_rebuild_verlet_list) { +#ifdef CALIPER + CALI_MARK_BEGIN("link_cell"); +#endif m_verlet_list.clear(); link_cell([&](Particle &p1, Particle &p2, Distance const &d) { @@ -734,7 +739,13 @@ struct CellStructure : public System::Leaf { m_rebuild_verlet_list = false; m_rebuild_cabana_verlet_list = true; +#ifdef CALIPER + CALI_MARK_END("link_cell"); +#endif } else { +#ifdef CALIPER + CALI_MARK_BEGIN("pair_kernel"); +#endif auto const maybe_box = decomposition().minimum_image_distance(); /* In this case the pair kernel is just run over the verlet list. */ if (maybe_box) { @@ -751,6 +762,9 @@ struct CellStructure : public System::Leaf { distance_function(*pair.first, *pair.second)); } } +#ifdef CALIPER + CALI_MARK_END("pair_kernel"); +#endif } } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 06e8702cdee..5f957de3637 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -38,41 +38,36 @@ class CustomVerletList // Custom constructor template CustomVerletList(PositionSlice x, const std::size_t begin, - const std::size_t end, const std::size_t max_neigh, - const std::size_t thread_number) { - initializeData(x.size(), max_neigh, thread_number); + const std::size_t end, const std::size_t max_neigh) { + //const std::size_t thread_number) { + initializeData(x.size(), max_neigh);//, thread_number); } virtual ~CustomVerletList() {}; -private: - Kokkos::View max_thread; - public: Kokkos::View counts; Kokkos::View neighbors; + // Kokkos::View neighbors; // Method to initialize _data without filling neighbors KOKKOS_INLINE_FUNCTION void initializeData(const std::size_t num_particles, - const std::size_t max_neigh, - const std::size_t thread_number) { + const std::size_t max_neigh) { + //const std::size_t thread_number) { counts = Kokkos::View("num_neighbors", num_particles); neighbors = Kokkos::View( + // neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, max_neigh); - max_thread = Kokkos::View("max_thread", thread_number); - for (int tid = 0; tid < thread_number; ++tid) { - max_thread(tid) = 1; - } } // Method to add a neighbor KOKKOS_INLINE_FUNCTION - void addNeighbor(const int tid, int pid, int nid) { + void addNeighbor(int pid, int nid) { std::size_t count = counts(pid); - std::size_t max_t = max_thread(tid); + std::size_t count_n = counts(nid); - if (count + 1 > max_t) { + if (count > count_n) { int tmp = pid; pid = nid; nid = tmp; @@ -80,35 +75,31 @@ class CustomVerletList count = Kokkos::atomic_fetch_add(&counts(pid), 1); if (count >= neighbors.extent(1)) { throw std::runtime_error( - "Number of count in one thread is larger than VerletList size."); + //Kokkos::abort( + "Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; - std::size_t new_count = count + 1; - if (new_count > max_t) - max_thread(tid) = new_count; } // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION - void addNeighborNonAtomic(const int tid, int pid, int nid) { + void addNeighborNonAtomic(int pid, int nid) { std::size_t count = counts(pid); - std::size_t max_t = max_thread(tid); + std::size_t count_n = counts(nid); - if (count + 1 > max_t) { + if (count > count_n) { int tmp = pid; pid = nid; nid = tmp; count = counts(pid); } if (count >= neighbors.extent(1)) { - // expandMaxNeighbors(neighbors.extent(1) * 2); throw std::runtime_error( + //Kokkos::abort( "Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; counts(pid) += 1; - if (count + 1 > max_thread(tid)) - max_thread(tid) = count + 1; } // Find max counts diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 9ec856d3a8a..2607a6d7ff0 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -136,7 +136,7 @@ void cabana_short_range( std::vector unique_particles; // std::vector sequential_particles; int index = 0; - // int max_id = 0; + int max_id = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); // if (rank == 0) { @@ -157,7 +157,7 @@ void cabana_short_range( for (auto &p : particles) { if (cell_structure.get_local_particle(p.id())) { - // if (p.id() > max_id) max_id = p.id(); + if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); // sequential_particles.emplace_back(p); @@ -168,7 +168,7 @@ void cabana_short_range( for (auto &p : ghost_particles) { if (not registered_index.contains(p.id())) { if (cell_structure.get_local_particle(p.id())) { - // if (p.id() > max_id) max_id = p.id(); + if (p.id() > max_id) max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); // sequential_particles.emplace_back(p); @@ -180,7 +180,7 @@ void cabana_short_range( // If we do not rebuild we can use the saved map index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); - // max_id = saved_data.get_max_id(); + max_id = saved_data.get_max_id(); } int number_of_unique_particles = index; @@ -194,6 +194,7 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Allocation"); #endif + Kokkos::View id_to_index("id_to_index", max_id + 1); Kokkos::View local_force( "local_force", num_threads, number_of_unique_particles, 3); @@ -212,9 +213,11 @@ void cabana_short_range( using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), - [&unique_particles, &aosoa, &box_l](const int p_id) { + //[&unique_particles, &aosoa, &box_l](const int p_id) { + [&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); + id_to_index(unique_particles.at(p_id)->id()) = p_id; }); Kokkos::fence(); @@ -446,13 +449,13 @@ void cabana_short_range( // std::cout << "max_counts:" << max_counts << " " << max_cutoff << // std::endl; if (rebuild) { // Legacy Velert List - /*verlet_list = - ListType(aosoa.position, 0, aosoa.position.size(), max_counts, - num_threads); auto kernel = [&](Particle const &p1, Particle const &p2) { - auto thread_id = omp_get_thread_num(); - verlet_list.addNeighbor(thread_id, - id_to_index(p1.id()), - id_to_index(p2.id())); + if (1) { + verlet_list = + ListType(aosoa.position, 0, aosoa.position.size(), max_counts); //, num_threads); + auto kernel = [&verlet_list, &id_to_index](Particle const &p1, Particle const &p2) { + verlet_list.addNeighborNonAtomic( + id_to_index(p1.id()), + id_to_index(p2.id())); //std::cout << "WITHSMP " //<< id_to_index(p1.id()) << " " //<< id_to_index(p2.id()) << " " @@ -465,8 +468,10 @@ void cabana_short_range( }; cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); - */ + // verlet_list.get_max_counts(); + } } else { + //if (not rebuild) { // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); } @@ -500,52 +505,113 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); #endif - if (1) { - verlet_list = create_verlet_list( + if (0) { + ListType v_verlet_list; + v_verlet_list = create_verlet_list( max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, // sequential_particles, verlet_criterion, first_neighbor_kernel, cell_structure); - /*using neighbor_list = Cabana::NeighborList; - std::vector> interaction_pairs; - - for (int i = 0; i < number_of_unique_particles; ++i) { - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - //interaction_pairs.emplace_back(i, j); - std::cout << "*Cabana* " - << i << " " - << j << " " - << aosoa.ghost(i) << " " - << aosoa.ghost(j) << " " - << aosoa.id(i) << " " - << aosoa.id(j) << "\n"; - } - }*/ // verlet_list.get_max_counts(); } #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List by Cabana"); #endif - } else { - //{ + } //else { + { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); #endif + //using neighbor_list = Cabana::NeighborList; + //std::vector> interaction_pairs; + //std::vector> interaction_pairs; /* - Kokkos::parallel_for( - "ForceLoop", Kokkos::RangePolicy<>(0,interaction_pairs.size()), - KOKKOS_LAMBDA(int idx) { - auto i = interaction_pairs[idx].first; - auto j = interaction_pairs[idx].second; - first_neighbor_kernel(i, j); - }); + for (int i = 0; i < number_of_unique_particles; ++i) { + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + first_neighbor_kernel(i, j); + //interaction_pairs.emplace_back(i, j); + //interaction_pairs.emplace_back(unique_particles.at(i), + // unique_particles.at(j)); + //std::cout << "*Cabana* " + // << i << " " + // << j << " " + // << aosoa.ghost(i) << " " + // << aosoa.ghost(j) << " " + // << aosoa.id(i) << " " + // << aosoa.id(j) << "\n"; + } + } */ // verlet_list.get_max_counts(); + /* + for (auto &pair : interaction_pairs) { + first_neighbor_kernel(pair.first, pair.second); + //auto p1 = pair.first; + //auto p2 = pair.second; + //int i = id_to_index(p1->id()); + //int j = id_to_index(p2->id()); + int i = pair.first; + int j = pair.second; + //auto p1 = unique_particles.at(i); + //auto p2 = unique_particles.at(j); + auto thread_id = omp_get_thread_num(); + + IA_parameters const &ia_params = + nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); + //nonbonded_ias.get_ia_param(p1->type(), p2->type()); + + ParticleForce pf{}; + Utils::Vector3d const pi = {aosoa.position(i, 0), aosoa.position(i, 1), + aosoa.position(i, 2)}; + Utils::Vector3d const pj = {aosoa.position(j, 0), aosoa.position(j, 1), + aosoa.position(j, 2)}; + + Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); + //Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), + // p2->pos()); + auto const dist = d.norm(); + + auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); + //auto const q1q2 = p1->q() * p2->q(); + + bool do_nonbonded_flag = true; + add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, + do_nonbonded_flag, coulomb_kernel); + + local_force(thread_id, i, 0) += pf.f[0]; + local_force(thread_id, i, 1) += pf.f[1]; + local_force(thread_id, i, 2) += pf.f[2]; + + auto opf = calc_opposing_force(pf, d); + local_force(thread_id, j, 0) += opf.f[0]; + local_force(thread_id, j, 1) += opf.f[1]; + local_force(thread_id, j, 2) += opf.f[2]; + } + */ + // Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, Cabana::FirstNeighborsTag(), Cabana::TeamOpTag()); + // + /* + using neighbor_list = Cabana::NeighborList; + using SimdPolicy = Cabana::SimdPolicy; + SimdPolicy simd_policy(0, (number_of_unique_particles - 1 + vector_length) / vector_length); + Cabana::simd_parallel_for(simd_policy, + [&number_of_unique_particles, &verlet_list, + &first_neighbor_kernel] (const int s, const int a) { + int i = s * vector_length + a; + if (i > number_of_unique_particles) return; + + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + first_neighbor_kernel(i, j); + } + }); + */ + Kokkos::fence(); #ifdef CALIPER @@ -557,7 +623,7 @@ void cabana_short_range( if (rebuild) { // CabanaData new_data(verlet_list, unique_particles, // unique_particles.size()); - CabanaData new_data(verlet_list, unique_particles); + CabanaData new_data(verlet_list, unique_particles, max_id); cell_structure.set_cabana_data(std::make_unique(new_data)); } diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp index 5d8aa082197..09d4f808ac1 100644 --- a/src/core/verlet_list_loop.hpp +++ b/src/core/verlet_list_loop.hpp @@ -83,6 +83,7 @@ inline int set_interacting_pair_cell( // ActiveProtocol le_protocol, Kokkos::View &bin_size, Cabana::LinkedCellList &cell_list, + //std::vector> &interacting_pair_cell) { Kokkos::View &interacting_pair_cell) { #ifdef CALIPER CALI_CXX_MARK_FUNCTION; @@ -167,6 +168,7 @@ inline int set_interacting_pair_cell( // interacting_pair_cell(pcid, 1) = cid_j; interacting_pair_cell(pair_cell_id, 0) = cid_i; interacting_pair_cell(pair_cell_id, 1) = cid_j; + // interacting_pair_cell.emplace_back(std::pair(cid_i, cid_j)); ++pair_cell_id; // interacting_pair_cell_thread(thread_id, pair_id_thread(thread_id), // 0) = cid_i; interacting_pair_cell_thread(thread_id, @@ -244,10 +246,9 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // Cabana::permute( cell_list, particle_storage ); // Number of threads - int num_threads = execution_space().concurrency(); - + //int num_threads = execution_space().concurrency(); ListType verlet_list = ListType(aosoa.position, 0, aosoa.position.size(), - max_counts, num_threads); + max_counts);//, num_threads); // Offset particle id and the number of particle in specific cell Kokkos::View bin_offset("bin_offset", total_bins); @@ -267,6 +268,8 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, } Kokkos::View interacting_pair_cell( "interacting_pair_cell", total_pair_cell - total_bins, 2); + //std::vector> interacting_pair_cell; + //interacting_pair_cell.reserve(total_pair_cell - total_bins); int empty_pair_number = set_interacting_pair_cell( total_bins, total_pair_cell, cell_num, delta_lebc, le_direction, le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); @@ -280,9 +283,9 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, // This kernel calculate within each cell auto kernel_each = [&bin_offset, &bin_size, &original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, - &distance_function, &verlet_list, - &first_neighbor_kernel](const int cid_i) { - auto thread_id = omp_get_thread_num(); + &distance_function, &verlet_list](const int cid_i) { + //&first_neighbor_kernel](const int cid_i) { + //auto thread_id = omp_get_thread_num(); int offset_i = bin_offset(cid_i); int size_i = bin_size(cid_i); @@ -308,8 +311,9 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, if (verlet_criterion(*p1, *unique_particles.at(jj), distance_function(*p1, *unique_particles.at(jj)))) { - verlet_list.addNeighborNonAtomic(thread_id, ii, jj); - first_neighbor_kernel(ii, jj); + //verlet_list.addNeighborNonAtomic(thread_id, ii, jj); + verlet_list.addNeighborNonAtomic(ii, jj); + //first_neighbor_kernel(ii, jj); } } } // i-loop @@ -319,12 +323,14 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, auto kernel_neighbor = [&interacting_pair_cell, &bin_offset, &bin_size, &original_idx, &aosoa_id, &aosoa_ghost, &unique_particles, &verlet_criterion, - &distance_function, &verlet_list, - &first_neighbor_kernel](const int pair_cell_i) { + &distance_function, &verlet_list](const int pair_cell_i) { + //&first_neighbor_kernel](const int pair_cell_i) { int cid_i = interacting_pair_cell(pair_cell_i, 0); int cid_j = interacting_pair_cell(pair_cell_i, 1); + // int cid_i = interacting_pair_cell.at(pair_cell_i).first; + // int cid_j = interacting_pair_cell.at(pair_cell_i).second; - auto thread_id = omp_get_thread_num(); + //auto thread_id = omp_get_thread_num(); int offset_i = bin_offset(cid_i); int size_i = bin_size(cid_i); @@ -353,8 +359,9 @@ ListType create_verlet_list(double const max_cutoff, int const max_counts, if (verlet_criterion(*p1, *unique_particles.at(jj), distance_function(*p1, *unique_particles.at(jj)))) { - verlet_list.addNeighbor(thread_id, ii, jj); - first_neighbor_kernel(ii, jj); + //verlet_list.addNeighbor(thread_id, ii, jj); + verlet_list.addNeighbor(ii, jj); + //first_neighbor_kernel(ii, jj); } } // i-loop } From 3e2d669f7122e076568adf2af38594ae7dbeb6ff Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 11 Jul 2025 19:29:45 +0200 Subject: [PATCH 51/94] Improved computation rate --- src/core/BoxGeometry.hpp | 33 +++ src/core/aosoa_pack.hpp | 2 +- src/core/cabana_data.hpp | 1 - src/core/cell_system/CellStructure.hpp | 4 +- src/core/custom_verlet_list.hpp | 8 +- src/core/short_range_cabana.hpp | 199 ++++++++----- src/core/verlet_list_loop.hpp | 382 ------------------------- 7 files changed, 176 insertions(+), 453 deletions(-) delete mode 100644 src/core/verlet_list_loop.hpp diff --git a/src/core/BoxGeometry.hpp b/src/core/BoxGeometry.hpp index b16fb3c48d8..0cc216f2888 100644 --- a/src/core/BoxGeometry.hpp +++ b/src/core/BoxGeometry.hpp @@ -225,6 +225,39 @@ class BoxGeometry { get_mi_coord(a[2], b[2], 2)}; } + /** + * @brief Get the minimum-image vector between two coordinates. + * + * @tparam T Floating point type. + * + * @param a0 x element of the terminal point. + * @param a1 y element of the terminal point. + * @param a2 z element of the terminal point. + * @param b0 x element of the initial point. + * @param b1 x element of the initial point. + * @param b2 x element of the initial point. + * @return Vector from @p b to @p a that minimizes the distance across + * periodic images, i.e. a - b. + */ + template + Utils::Vector get_mi_vector(const T &a0, const T &a1, const T &a2, + const T &b0, const T &b1, const T &b2) const { + if (type() == BoxType::LEES_EDWARDS) { + auto const shear_plane_normal = lees_edwards_bc().shear_plane_normal; + auto a_tmp = Utils::Vector {a0, a1, a2}; + auto b_tmp = Utils::Vector {b0, b1, b2}; + a_tmp[shear_plane_normal] = Algorithm::periodic_fold( + a_tmp[shear_plane_normal], m_length[shear_plane_normal]); + b_tmp[shear_plane_normal] = Algorithm::periodic_fold( + b_tmp[shear_plane_normal], m_length[shear_plane_normal]); + return lees_edwards_bc().distance(a_tmp - b_tmp, m_length, m_length_half, + m_length_inv, m_periodic); + } + assert(type() == BoxType::CUBOID); + return {get_mi_coord(a0, b0, 0), get_mi_coord(a1, b1, 1), + get_mi_coord(a2, b2, 2)}; + } + BoxType type() const { return m_type; } void set_type(BoxType type) { m_type = type; } diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index 5987f6787b8..d745bb82492 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -24,7 +24,7 @@ #include const int vector_length = 1; -using data_types = Cabana::MemberTypes; +using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using AoSoA_type = Cabana::AoSoA; diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 537ca6bc5c8..869e5dfd89a 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -23,7 +23,6 @@ #include "custom_verlet_list.hpp" #include -#include #include using memory_space = Kokkos::SharedSpace; diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 2cff65e54da..f01885a634d 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -696,8 +696,8 @@ struct CellStructure : public System::Leaf { template void cabana_verlet_list_loop(Kernel kernel, const VerletCriterion &verlet_criterion) { - // if (m_rebuild_cabana_verlet_list) { - if (m_rebuild_verlet_list) { + if (m_rebuild_cabana_verlet_list) { + // if (m_rebuild_verlet_list) { m_verlet_list.clear(); link_cell([&](Particle &p1, Particle &p2, Distance const &d) { diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 5f957de3637..e9c779c3746 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -36,11 +36,13 @@ class CustomVerletList CustomVerletList() : Base() {} // Custom constructor - template - CustomVerletList(PositionSlice x, const std::size_t begin, + //template + //CustomVerletList(PositionSlice x, const std::size_t begin, + CustomVerletList(const std::size_t begin, const std::size_t end, const std::size_t max_neigh) { //const std::size_t thread_number) { - initializeData(x.size(), max_neigh);//, thread_number); + //initializeData(x.size(), max_neigh);//, thread_number); + initializeData(end - begin, max_neigh);//, thread_number); } virtual ~CustomVerletList() {}; diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 2607a6d7ff0..d4c8c725071 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -33,7 +33,7 @@ #include "aosoa_pack.hpp" #include "cabana_data.hpp" #include "custom_verlet_list.hpp" -#include "verlet_list_loop.hpp" +//#include "verlet_list_loop.hpp" #include #include #include @@ -138,10 +138,10 @@ void cabana_short_range( int index = 0; int max_id = 0; - bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list(); + bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); // if (rank == 0) { - // std::cout << "For CABANA rebuild " << rebuild - // << " " << Kokkos::OpenMP::concurrency() << std::endl; + // std::cout << "\nFor CABANA rebuild " << rebuild + // << " " << rank << std::endl; // } CabanaData saved_data; @@ -194,6 +194,7 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Allocation"); #endif + Kokkos::View id_to_index("id_to_index", max_id + 1); Kokkos::View local_force( "local_force", num_threads, number_of_unique_particles, 3); @@ -210,7 +211,6 @@ void cabana_short_range( // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); - using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), //[&unique_particles, &aosoa, &box_l](const int p_id) { @@ -219,7 +219,42 @@ void cabana_short_range( aosoa, box_l); id_to_index(unique_particles.at(p_id)->id()) = p_id; }); + Kokkos::fence(); + // After ONLY JUST creating LinkedCellList, force calculation became slower, + // even if it is not used and It is explicitly deleted. + if (0) + { + //Cabana::LinkedCellList cell_list; + double grid_min[3] = {0.0, 0.0, 0.0}; + double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; + double grid_delta[3] = {}; + int cell_num[3] = {}; + double eff_cutoff; + for (int d = 0; d < 3; ++d) { + eff_cutoff = pair_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + } + auto *cell_list = new Cabana::LinkedCellList( + aosoa.position, grid_delta, grid_min, grid_max); + // Now permute the AoSoA (i.e. reorder the data) + Cabana::permute( *cell_list, particle_storage ); + unique_particles.clear(); + for (int i = 0; i < aosoa.id.size(); ++i) { + id_to_index(aosoa.id(i)) = i; + unique_particles.emplace_back(cell_structure.get_local_particle(aosoa.id(i))); + } + delete cell_list; + Kokkos::fence(); + /*Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), + [&unique_particles, &aosoa, &box_l](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, + aosoa, box_l); + });*/ + } #ifdef CALIPER CALI_MARK_END("Cabana - Allocation"); @@ -332,12 +367,9 @@ void cabana_short_range( #ifdef NPT Utils::Vector3d virial{}; #endif - Utils::Vector3d const pi = {aosoa.position(i, 0), aosoa.position(i, 1), - aosoa.position(i, 2)}; - Utils::Vector3d const pj = {aosoa.position(j, 0), aosoa.position(j, 1), - aosoa.position(j, 2)}; - - Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); + Utils::Vector3d const d = box_geo.get_mi_vector( + aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), + aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); auto const dist = d.norm(); auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); @@ -449,11 +481,10 @@ void cabana_short_range( // std::cout << "max_counts:" << max_counts << " " << max_cutoff << // std::endl; if (rebuild) { // Legacy Velert List - if (1) { - verlet_list = - ListType(aosoa.position, 0, aosoa.position.size(), max_counts); //, num_threads); + if (0) { + verlet_list = ListType(0, number_of_unique_particles, max_counts); auto kernel = [&verlet_list, &id_to_index](Particle const &p1, Particle const &p2) { - verlet_list.addNeighborNonAtomic( + verlet_list.addNeighbor( id_to_index(p1.id()), id_to_index(p2.id())); //std::cout << "WITHSMP " @@ -505,12 +536,67 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); #endif - if (0) { - ListType v_verlet_list; - v_verlet_list = create_verlet_list( + if (1) { + //ListType v_verlet_list; + /* + verlet_list = create_verlet_list( max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, - // sequential_particles, verlet_criterion, first_neighbor_kernel, cell_structure); + */ + verlet_list = ListType(0, number_of_unique_particles, max_counts); + auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; + + auto kernel_each = [&cells, &distance_function, + &verlet_criterion, &id_to_index, &verlet_list, + max_id, &first_neighbor_kernel] (int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { + auto &p1 = *it; + if (p1.id() > max_id) continue; + /* Pairs in this cell */ + for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { + if ((*jt).id() > max_id) continue; + if (verlet_criterion(p1, *jt, + distance_function(p1, *jt))) { + int ii = id_to_index(p1.id()); + int jj = id_to_index((*jt).id()); + verlet_list.addNeighborNonAtomic(ii, jj); + //first_neighbor_kernel(ii, jj); + } + } + } + }; + + auto kernel_neighbor = [&cells, &distance_function, + &verlet_criterion, &id_to_index, &verlet_list, + max_id, &first_neighbor_kernel] (int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { + auto &p1 = *it; + if (p1.id() > max_id) continue; + /* Pairs with neighbors */ + for (auto &neighbor : cells[i]->neighbors().red()) { + for (auto &p2 : neighbor->particles()) { + if (p2.id() > max_id) continue; + if (verlet_criterion(p1, p2, + distance_function(p1, p2))) { + int ii = id_to_index(p1.id()); + int jj = id_to_index(p2.id()); + verlet_list.addNeighbor(ii, jj); + //first_neighbor_kernel(ii, jj); + } + } + } + } + }; + + Kokkos::parallel_for("each", cells.size(), kernel_each); + Kokkos::fence(); + + Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); + Kokkos::fence(); // verlet_list.get_max_counts(); } #ifdef CALIPER @@ -528,65 +614,50 @@ void cabana_short_range( for (int i = 0; i < number_of_unique_particles; ++i) { for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = neighbor_list::getNeighbor(verlet_list, i, n); - first_neighbor_kernel(i, j); - //interaction_pairs.emplace_back(i, j); + //first_neighbor_kernel(i, j); + interaction_pairs.emplace_back(i, j); //interaction_pairs.emplace_back(unique_particles.at(i), // unique_particles.at(j)); - //std::cout << "*Cabana* " - // << i << " " - // << j << " " - // << aosoa.ghost(i) << " " - // << aosoa.ghost(j) << " " - // << aosoa.id(i) << " " - // << aosoa.id(j) << "\n"; } } */ // verlet_list.get_max_counts(); /* - for (auto &pair : interaction_pairs) { - first_neighbor_kernel(pair.first, pair.second); - //auto p1 = pair.first; - //auto p2 = pair.second; - //int i = id_to_index(p1->id()); - //int j = id_to_index(p2->id()); - int i = pair.first; - int j = pair.second; - //auto p1 = unique_particles.at(i); - //auto p2 = unique_particles.at(j); - auto thread_id = omp_get_thread_num(); + // Essentially same as legacy ESPRESSO + for (int i = 0; i < number_of_unique_particles; ++i) { + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + //first_neighbor_kernel(i, j); + auto thread_id = omp_get_thread_num(); - IA_parameters const &ia_params = - nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); - //nonbonded_ias.get_ia_param(p1->type(), p2->type()); + IA_parameters const &ia_params = + nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); + //nonbonded_ias.get_ia_param(p1->type(), p2->type()); - ParticleForce pf{}; - Utils::Vector3d const pi = {aosoa.position(i, 0), aosoa.position(i, 1), - aosoa.position(i, 2)}; - Utils::Vector3d const pj = {aosoa.position(j, 0), aosoa.position(j, 1), - aosoa.position(j, 2)}; - - Utils::Vector3d const d = box_geo.get_mi_vector(pi, pj); - //Utils::Vector3d const d = box_geo.get_mi_vector(p1->pos(), - // p2->pos()); - auto const dist = d.norm(); + ParticleForce pf{}; - auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); - //auto const q1q2 = p1->q() * p2->q(); + Utils::Vector3d const d = box_geo.get_mi_vector( + aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), + aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); - bool do_nonbonded_flag = true; + auto const dist = d.norm(); - add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, - do_nonbonded_flag, coulomb_kernel); + auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); - local_force(thread_id, i, 0) += pf.f[0]; - local_force(thread_id, i, 1) += pf.f[1]; - local_force(thread_id, i, 2) += pf.f[2]; + bool do_nonbonded_flag = true; - auto opf = calc_opposing_force(pf, d); - local_force(thread_id, j, 0) += opf.f[0]; - local_force(thread_id, j, 1) += opf.f[1]; - local_force(thread_id, j, 2) += opf.f[2]; + add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, + do_nonbonded_flag, coulomb_kernel); + + local_force(thread_id, i, 0) += pf.f[0]; + local_force(thread_id, i, 1) += pf.f[1]; + local_force(thread_id, i, 2) += pf.f[2]; + + auto opf = calc_opposing_force(pf, d); + local_force(thread_id, j, 0) += opf.f[0]; + local_force(thread_id, j, 1) += opf.f[1]; + local_force(thread_id, j, 2) += opf.f[2]; + } } */ // diff --git a/src/core/verlet_list_loop.hpp b/src/core/verlet_list_loop.hpp deleted file mode 100644 index 09d4f808ac1..00000000000 --- a/src/core/verlet_list_loop.hpp +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Copyright (C) 2010-2025 The ESPResSo project - * - * This file is part of ESPResSo. - * - * ESPResSo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * ESPResSo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include "config/config.hpp" - -#include "cell_system/CellStructure.hpp" -#include "lees_edwards/lees_edwards.hpp" - -#ifdef CALIPER -#include -#endif - -#ifdef SHARED_MEMORY_PARALLELISM - -#include "aosoa_pack.hpp" -#include "cabana_data.hpp" -#include "custom_verlet_list.hpp" -#include -#include -#include -#include -#include -#include -#include - -using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; -using execution_space = Kokkos::DefaultExecutionSpace; - -inline void set_offset_and_size_indexed_by_cid( - int &total_bins, int *cell_num, - Cabana::LinkedCellList &cell_list, - Kokkos::View &bin_offset, - Kokkos::View &bin_size, - Kokkos::View &original_idx) { -#ifdef CALIPER - CALI_CXX_MARK_FUNCTION; -#endif - // for (int cid = 0; cid < total_bins; ++cid) { - Kokkos::parallel_for( - "set_offset", total_bins, - [&cell_num, &cell_list, &bin_offset, &bin_size](const int cid) { - int dx[3] = {}; - dx[0] = static_cast(cid / (cell_num[1] * cell_num[2])); - dx[1] = static_cast((cid - dx[0] * (cell_num[1] * cell_num[2])) / - cell_num[2]); - dx[2] = cid % cell_num[2]; - bin_offset(cid) = cell_list.binOffset(dx[0], dx[1], dx[2]); - bin_size(cid) = cell_list.binSize(dx[0], dx[1], dx[2]); - - // Calculate particle_bins - cell_list(cid); - }); - Kokkos::parallel_for("set_permutation", original_idx.extent(0), - [&cell_list, &original_idx](const int i) { - original_idx(i) = cell_list.permutation(i); - }); -} - -using ActiveProtocol = std::variant; -inline int set_interacting_pair_cell( - int &total_bins, int total_pair_cell, int *cell_num, int *delta_lebc, - int le_direction, int le_normal, - std::shared_ptr le_protocol, - // ActiveProtocol le_protocol, - Kokkos::View &bin_size, - Cabana::LinkedCellList &cell_list, - //std::vector> &interacting_pair_cell) { - Kokkos::View &interacting_pair_cell) { -#ifdef CALIPER - CALI_CXX_MARK_FUNCTION; -#endif - constexpr int ijkIndexes[27][3] = { - {-1, -1, -1}, {-1, -1, 0}, {-1, -1, 1}, {-1, 0, -1}, {-1, 0, 0}, - {-1, 0, 1}, {-1, 1, -1}, {-1, 1, 0}, {-1, 1, 1}, {0, -1, -1}, - {0, -1, 0}, {0, -1, 1}, {0, 0, -1}, {0, 0, 0}, {0, 0, 1}, - {0, 1, -1}, {0, 1, 0}, {0, 1, 1}, {1, -1, -1}, {1, -1, 0}, - {1, -1, 1}, {1, 0, -1}, {1, 0, 0}, {1, 0, 1}, {1, 1, -1}, - {1, 1, 0}, {1, 1, 1}}; - - int empty_pair_number = 0; - int pair_cell_id = 0; - - for (int cid_i = 0; cid_i < total_bins; ++cid_i) { - // Kokkos::parallel_for("set_interacting_pair_cell", total_bins, - // KOKKOS_LAMBDA(const int cid_i) { - // auto thread_id = omp_get_thread_num(); - // Obtaining 3 dimentional cell index from cid_i - int index[3] = {}; - cell_list.ijkBinIndex(cid_i, index[0], index[1], index[2]); - int dx[3]; - // From 27 neighbor cell, the list of interacting pair cell is created - for (int n = 0; n < 27; ++n) { - - if (le_protocol == nullptr) { - if (index[0] != 0 and ijkIndexes[n][0] == -1) - continue; - - if (index[1] != 0 and ijkIndexes[n][1] == -1 and ijkIndexes[n][0] == 0) - continue; - } - - bool duplicate_cell = false; - // Obtaining 3 dimentional cell index from neighbor cell - for (int d = 0; d < 3; ++d) { - dx[d] = (ijkIndexes[n][d] + index[d] + cell_num[d]) % cell_num[d]; - if (cell_num[d] <= 2 and ijkIndexes[n][d] + index[d] != dx[d]) - duplicate_cell = true; - } - if (duplicate_cell) - continue; - - // Lees-Edwards BC - int le_crossing = 0; - if (le_protocol != nullptr) { - le_crossing = - ijkIndexes[n][le_normal] + index[le_normal] - dx[le_normal]; - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - delta_lebc[le_direction] + - cell_num[le_direction]) % - cell_num[le_direction]; - } - } - // Additional Cell - /* - if (le_crossing != 0 && index[le_direction] == 1) { - if (le_crossing < 0) { - dx[le_direction] = (dx[le_direction] + 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } else if (le_crossing > 0) { - dx[le_direction] = (dx[le_direction] - 1 + - cell_num[le_direction]) % cell_num[le_direction]; - } - cell_offset = bin_offset(dx[0], dx[1], dx[2]); - cell_size = bin_size(dx[0], dx[1], dx[2]); - } - */ - - // Interacting pair cell is registered in the list - int cid_j = cell_list.cardinalBinIndex(dx[0], dx[1], dx[2]); - // if (cid_i <= cid_j) { - if (cid_i < cid_j) { - if (bin_size(cid_i) != 0 and bin_size(cid_j) != 0) { - // std::size_t pcid = Kokkos::atomic_fetch_inc(&pair_cell_id()); - // interacting_pair_cell(pcid, 0) = cid_i; - // interacting_pair_cell(pcid, 1) = cid_j; - interacting_pair_cell(pair_cell_id, 0) = cid_i; - interacting_pair_cell(pair_cell_id, 1) = cid_j; - // interacting_pair_cell.emplace_back(std::pair(cid_i, cid_j)); - ++pair_cell_id; - // interacting_pair_cell_thread(thread_id, pair_id_thread(thread_id), - // 0) = cid_i; interacting_pair_cell_thread(thread_id, - // pair_id_thread(thread_id), 1) = cid_j; pair_id_thread(thread_id) += - // 1; - } else { - // Kokkos::atomic_inc(&empty_pair_number()); - ++empty_pair_number; - // empty_thread(thread_id) += 1; - } - } - } - //}); - } - return empty_pair_number; -} - -using ListAlgorithm = Cabana::HalfNeighborTag; -using ListType = Cabana::CustomVerletList; -template -ListType create_verlet_list(double const max_cutoff, int const max_counts, - AoSoA_pack &aosoa, - std::vector &unique_particles, - VerletCriterion const &verlet_criterion, - Kernel &first_neighbor_kernel, - CellStructure &cell_structure) { - // Creating LinkedCellList and VerletList: - // Box Properties - auto const &system = ::System::get_system(); - auto box_geo = *(system.box_geo); - auto box_l = box_geo.length(); - Cabana::LinkedCellList cell_list; - double grid_min[3] = {0.0, 0.0, 0.0}; - double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; - double grid_delta[3] = {}; - int cell_num[3] = {}; - double eff_cutoff; - for (int d = 0; d < 3; ++d) { - eff_cutoff = max_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); - } - // For Lees-Edwards boundary condition - double le_offset; - int le_direction; - int le_normal; - int delta_lebc[3] = {0, 0, 0}; - auto le_protocol = system.lees_edwards->get_protocol(); - if (le_protocol == nullptr) { - le_offset = 0.; - le_direction = -1; - le_normal = -1; - } else { - le_offset = box_geo.lees_edwards_bc().pos_offset; - le_direction = box_geo.lees_edwards_bc().shear_direction; - le_normal = box_geo.lees_edwards_bc().shear_plane_normal; - delta_lebc[le_direction] = - static_cast(std::ceil(le_offset / grid_delta[le_direction])) % - cell_num[le_direction]; - } -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - CellList"); -#endif - cell_list = Cabana::createLinkedCellList( - aosoa.position, grid_delta, grid_min, grid_max); -#ifdef CALIPER - CALI_MARK_END("Cabana - CellList"); -#endif - int total_bins = cell_list.totalBins(); - // Now permute the AoSoA (i.e. reorder the data) using the linked cell - // list. - // Cabana::permute( cell_list, particle_storage ); - - // Number of threads - //int num_threads = execution_space().concurrency(); - ListType verlet_list = ListType(aosoa.position, 0, aosoa.position.size(), - max_counts);//, num_threads); - - // Offset particle id and the number of particle in specific cell - Kokkos::View bin_offset("bin_offset", total_bins); - Kokkos::View bin_size("bin_size", total_bins); - Kokkos::View original_idx("original_idx", - aosoa.position.size()); - set_offset_and_size_indexed_by_cid(total_bins, cell_num, cell_list, - bin_offset, bin_size, original_idx); - auto const particle_bins = cell_list.getParticleBins(); - - // Creating Interacting cell - int total_pair_cell; - if (total_bins < 27) { - total_pair_cell = (total_bins - 1) * total_bins / 2 + total_bins; - } else { - total_pair_cell = 14 * total_bins; - } - Kokkos::View interacting_pair_cell( - "interacting_pair_cell", total_pair_cell - total_bins, 2); - //std::vector> interacting_pair_cell; - //interacting_pair_cell.reserve(total_pair_cell - total_bins); - int empty_pair_number = set_interacting_pair_cell( - total_bins, total_pair_cell, cell_num, delta_lebc, le_direction, - le_normal, le_protocol, bin_size, cell_list, interacting_pair_cell); - - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - - auto aosoa_id = aosoa.id; - auto aosoa_ghost = aosoa.ghost; - - // This kernel calculate within each cell - auto kernel_each = [&bin_offset, &bin_size, &original_idx, &aosoa_id, - &aosoa_ghost, &unique_particles, &verlet_criterion, - &distance_function, &verlet_list](const int cid_i) { - //&first_neighbor_kernel](const int cid_i) { - //auto thread_id = omp_get_thread_num(); - - int offset_i = bin_offset(cid_i); - int size_i = bin_size(cid_i); - - for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; - int ii = original_idx(i); // get previous id - int id_i = aosoa_id(ii); - auto p1 = unique_particles.at(ii); - // auto p1 = cell_structure.get_local_particle(id_i); - for (int j = i + 1; j < offset_i + size_i; ++j) { - // int jj = j; - int jj = original_idx(j); - int id_j = aosoa_id(jj); - if (aosoa_ghost(ii) or aosoa_ghost(jj)) { - if (((id_i < id_j) and aosoa_ghost(ii)) or - ((id_i > id_j) and aosoa_ghost(jj))) { - continue; - } - } - // auto p2 = unique_particles.at(jj); - // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *unique_particles.at(jj), - distance_function(*p1, - *unique_particles.at(jj)))) { - //verlet_list.addNeighborNonAtomic(thread_id, ii, jj); - verlet_list.addNeighborNonAtomic(ii, jj); - //first_neighbor_kernel(ii, jj); - } - } - } // i-loop - }; - - // This kernel used the loop for the pair of interacting cell - auto kernel_neighbor = [&interacting_pair_cell, &bin_offset, &bin_size, - &original_idx, &aosoa_id, &aosoa_ghost, - &unique_particles, &verlet_criterion, - &distance_function, &verlet_list](const int pair_cell_i) { - //&first_neighbor_kernel](const int pair_cell_i) { - int cid_i = interacting_pair_cell(pair_cell_i, 0); - int cid_j = interacting_pair_cell(pair_cell_i, 1); - // int cid_i = interacting_pair_cell.at(pair_cell_i).first; - // int cid_j = interacting_pair_cell.at(pair_cell_i).second; - - //auto thread_id = omp_get_thread_num(); - - int offset_i = bin_offset(cid_i); - int size_i = bin_size(cid_i); - int offset_j = bin_offset(cid_j); - int size_j = bin_size(cid_j); - - for (int i = offset_i; i < offset_i + size_i; ++i) { - // int ii = i; - int ii = original_idx(i); // get previous id - int id_i = aosoa_id(ii); - auto p1 = unique_particles.at(ii); - // auto p1 = cell_structure.get_local_particle(id_i); - - for (int j = offset_j; j < offset_j + size_j; ++j) { - // int jj = j; - int jj = original_idx(j); - int id_j = aosoa_id(jj); - if (aosoa_ghost(ii) or aosoa_ghost(jj)) { - if (((id_i < id_j) and aosoa_ghost(ii)) or - ((id_i > id_j) and aosoa_ghost(jj))) { - continue; - } - } - // auto p2 = unique_particles.at(jj); - // auto p2 = cell_structure.get_local_particle(id_j); - if (verlet_criterion(*p1, *unique_particles.at(jj), - distance_function(*p1, - *unique_particles.at(jj)))) { - //verlet_list.addNeighbor(thread_id, ii, jj); - verlet_list.addNeighbor(ii, jj); - //first_neighbor_kernel(ii, jj); - } - } // i-loop - } - }; - - Kokkos::RangePolicy policy_each(0, total_bins); - Kokkos::parallel_for("calc_by_cell_list_each", policy_each, kernel_each); - Kokkos::fence(); - - Kokkos::RangePolicy policy_neighbor( - 0, total_pair_cell - total_bins - empty_pair_number); - Kokkos::parallel_for("calc_by_cell_list_beighbor", policy_neighbor, - kernel_neighbor); - Kokkos::fence(); - - return verlet_list; -} -#endif // SHARED_MEMORY_PARALLELISM From 802def733e1633931adc10d585fba156aa5b7e35 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 11 Jul 2025 19:35:10 +0200 Subject: [PATCH 52/94] Formatting --- src/core/BoxGeometry.hpp | 7 +- src/core/cabana_data.hpp | 6 +- src/core/cell_system/CellStructure.hpp | 10 +- src/core/custom_verlet_list.hpp | 22 +- src/core/short_range_cabana.hpp | 332 +++++++++++-------------- 5 files changed, 174 insertions(+), 203 deletions(-) diff --git a/src/core/BoxGeometry.hpp b/src/core/BoxGeometry.hpp index 0cc216f2888..9f90836dd03 100644 --- a/src/core/BoxGeometry.hpp +++ b/src/core/BoxGeometry.hpp @@ -241,11 +241,12 @@ class BoxGeometry { */ template Utils::Vector get_mi_vector(const T &a0, const T &a1, const T &a2, - const T &b0, const T &b1, const T &b2) const { + const T &b0, const T &b1, + const T &b2) const { if (type() == BoxType::LEES_EDWARDS) { auto const shear_plane_normal = lees_edwards_bc().shear_plane_normal; - auto a_tmp = Utils::Vector {a0, a1, a2}; - auto b_tmp = Utils::Vector {b0, b1, b2}; + auto a_tmp = Utils::Vector{a0, a1, a2}; + auto b_tmp = Utils::Vector{b0, b1, b2}; a_tmp[shear_plane_normal] = Algorithm::periodic_fold( a_tmp[shear_plane_normal], m_length[shear_plane_normal]); b_tmp[shear_plane_normal] = Algorithm::periodic_fold( diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 869e5dfd89a..7a9bb5e1305 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -43,9 +43,9 @@ class CabanaData { CabanaData(ListType verlet_list, std::vector unique_particles) : verlet_list(verlet_list), unique_particles(unique_particles) {} CabanaData(ListType verlet_list, std::vector unique_particles, - int max_id) - : verlet_list(verlet_list), unique_particles(unique_particles), - max_id(max_id) {} + int max_id) + : verlet_list(verlet_list), unique_particles(unique_particles), + max_id(max_id) {} ListType get_verlet_list() const { return verlet_list; } int get_index() const { return unique_particles.size(); } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index f01885a634d..39d5547bebb 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -697,7 +697,7 @@ struct CellStructure : public System::Leaf { void cabana_verlet_list_loop(Kernel kernel, const VerletCriterion &verlet_criterion) { if (m_rebuild_cabana_verlet_list) { - // if (m_rebuild_verlet_list) { + // if (m_rebuild_verlet_list) { m_verlet_list.clear(); link_cell([&](Particle &p1, Particle &p2, Distance const &d) { @@ -726,7 +726,7 @@ struct CellStructure : public System::Leaf { * we go. */ if (m_rebuild_verlet_list) { #ifdef CALIPER - CALI_MARK_BEGIN("link_cell"); + CALI_MARK_BEGIN("link_cell"); #endif m_verlet_list.clear(); @@ -740,11 +740,11 @@ struct CellStructure : public System::Leaf { m_rebuild_verlet_list = false; m_rebuild_cabana_verlet_list = true; #ifdef CALIPER - CALI_MARK_END("link_cell"); + CALI_MARK_END("link_cell"); #endif } else { #ifdef CALIPER - CALI_MARK_BEGIN("pair_kernel"); + CALI_MARK_BEGIN("pair_kernel"); #endif auto const maybe_box = decomposition().minimum_image_distance(); /* In this case the pair kernel is just run over the verlet list. */ @@ -763,7 +763,7 @@ struct CellStructure : public System::Leaf { } } #ifdef CALIPER - CALI_MARK_END("pair_kernel"); + CALI_MARK_END("pair_kernel"); #endif } } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index e9c779c3746..4d1155845fa 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -36,13 +36,13 @@ class CustomVerletList CustomVerletList() : Base() {} // Custom constructor - //template - //CustomVerletList(PositionSlice x, const std::size_t begin, - CustomVerletList(const std::size_t begin, - const std::size_t end, const std::size_t max_neigh) { - //const std::size_t thread_number) { - //initializeData(x.size(), max_neigh);//, thread_number); - initializeData(end - begin, max_neigh);//, thread_number); + // template + // CustomVerletList(PositionSlice x, const std::size_t begin, + CustomVerletList(const std::size_t begin, const std::size_t end, + const std::size_t max_neigh) { + // const std::size_t thread_number) { + // initializeData(x.size(), max_neigh);//, thread_number); + initializeData(end - begin, max_neigh); //, thread_number); } virtual ~CustomVerletList() {}; @@ -55,10 +55,10 @@ class CustomVerletList KOKKOS_INLINE_FUNCTION void initializeData(const std::size_t num_particles, const std::size_t max_neigh) { - //const std::size_t thread_number) { + // const std::size_t thread_number) { counts = Kokkos::View("num_neighbors", num_particles); neighbors = Kokkos::View( - // neighbors = Kokkos::View( + // neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, max_neigh); } @@ -77,7 +77,7 @@ class CustomVerletList count = Kokkos::atomic_fetch_add(&counts(pid), 1); if (count >= neighbors.extent(1)) { throw std::runtime_error( - //Kokkos::abort( + // Kokkos::abort( "Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; @@ -97,7 +97,7 @@ class CustomVerletList } if (count >= neighbors.extent(1)) { throw std::runtime_error( - //Kokkos::abort( + // Kokkos::abort( "Number of count is larger than VerletList size."); } neighbors(pid, count) = nid; diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index d4c8c725071..502ce56d082 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -33,7 +33,7 @@ #include "aosoa_pack.hpp" #include "cabana_data.hpp" #include "custom_verlet_list.hpp" -//#include "verlet_list_loop.hpp" +// #include "verlet_list_loop.hpp" #include #include #include @@ -138,7 +138,8 @@ void cabana_short_range( int index = 0; int max_id = 0; - bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); + bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or + (not cell_structure.use_verlet_list); // if (rank == 0) { // std::cout << "\nFor CABANA rebuild " << rebuild // << " " << rank << std::endl; @@ -157,7 +158,8 @@ void cabana_short_range( for (auto &p : particles) { if (cell_structure.get_local_particle(p.id())) { - if (p.id() > max_id) max_id = p.id(); + if (p.id() > max_id) + max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); // sequential_particles.emplace_back(p); @@ -168,7 +170,8 @@ void cabana_short_range( for (auto &p : ghost_particles) { if (not registered_index.contains(p.id())) { if (cell_structure.get_local_particle(p.id())) { - if (p.id() > max_id) max_id = p.id(); + if (p.id() > max_id) + max_id = p.id(); registered_index.insert(p.id()); unique_particles.emplace_back(&p); // sequential_particles.emplace_back(p); @@ -212,48 +215,49 @@ void cabana_short_range( auto aosoa = AoSoA_pack(particle_storage); auto box_l = box_geo.length(); using policy_type = Kokkos::RangePolicy; - Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), - //[&unique_particles, &aosoa, &box_l](const int p_id) { - [&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, - aosoa, box_l); - id_to_index(unique_particles.at(p_id)->id()) = p_id; - }); + Kokkos::parallel_for( + "AoSoA write", policy_type(0, particle_storage.size()), + //[&unique_particles, &aosoa, &box_l](const int p_id) { + [&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); + id_to_index(unique_particles.at(p_id)->id()) = p_id; + }); Kokkos::fence(); // After ONLY JUST creating LinkedCellList, force calculation became slower, // even if it is not used and It is explicitly deleted. - if (0) - { - //Cabana::LinkedCellList cell_list; - double grid_min[3] = {0.0, 0.0, 0.0}; - double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; - double grid_delta[3] = {}; - int cell_num[3] = {}; - double eff_cutoff; - for (int d = 0; d < 3; ++d) { - eff_cutoff = pair_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); - } - auto *cell_list = new Cabana::LinkedCellList( - aosoa.position, grid_delta, grid_min, grid_max); - // Now permute the AoSoA (i.e. reorder the data) - Cabana::permute( *cell_list, particle_storage ); - unique_particles.clear(); - for (int i = 0; i < aosoa.id.size(); ++i) { - id_to_index(aosoa.id(i)) = i; - unique_particles.emplace_back(cell_structure.get_local_particle(aosoa.id(i))); - } - delete cell_list; - Kokkos::fence(); - /*Kokkos::parallel_for("AoSoA write", policy_type(0, particle_storage.size()), - [&unique_particles, &aosoa, &box_l](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, - aosoa, box_l); - });*/ + if (0) { + // Cabana::LinkedCellList cell_list; + double grid_min[3] = {0.0, 0.0, 0.0}; + double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; + double grid_delta[3] = {}; + int cell_num[3] = {}; + double eff_cutoff; + for (int d = 0; d < 3; ++d) { + eff_cutoff = pair_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + } + auto *cell_list = new Cabana::LinkedCellList( + aosoa.position, grid_delta, grid_min, grid_max); + // Now permute the AoSoA (i.e. reorder the data) + Cabana::permute(*cell_list, particle_storage); + unique_particles.clear(); + for (int i = 0; i < aosoa.id.size(); ++i) { + id_to_index(aosoa.id(i)) = i; + unique_particles.emplace_back( + cell_structure.get_local_particle(aosoa.id(i))); + } + delete cell_list; + Kokkos::fence(); + /*Kokkos::parallel_for("AoSoA write", policy_type(0, + particle_storage.size()), + [&unique_particles, &aosoa, &box_l](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, + aosoa, box_l); + });*/ } #ifdef CALIPER @@ -367,9 +371,9 @@ void cabana_short_range( #ifdef NPT Utils::Vector3d virial{}; #endif - Utils::Vector3d const d = box_geo.get_mi_vector( - aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), - aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); + Utils::Vector3d const d = box_geo.get_mi_vector( + aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), + aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); auto const dist = d.norm(); auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); @@ -482,28 +486,27 @@ void cabana_short_range( // std::endl; if (rebuild) { // Legacy Velert List if (0) { - verlet_list = ListType(0, number_of_unique_particles, max_counts); - auto kernel = [&verlet_list, &id_to_index](Particle const &p1, Particle const &p2) { - verlet_list.addNeighbor( - id_to_index(p1.id()), - id_to_index(p2.id())); - //std::cout << "WITHSMP " - //<< id_to_index(p1.id()) << " " - //<< id_to_index(p2.id()) << " " - //<< p1.is_ghost() << " " - //<< p2.is_ghost() << " " - //<< p1.id() << " " - //<< p2.id() << std::endl; - //<< p1.pos() << " " - //<< p2.pos() << "\n"; + verlet_list = ListType(0, number_of_unique_particles, max_counts); + auto kernel = [&verlet_list, &id_to_index](Particle const &p1, + Particle const &p2) { + verlet_list.addNeighbor(id_to_index(p1.id()), id_to_index(p2.id())); + // std::cout << "WITHSMP " + //<< id_to_index(p1.id()) << " " + //<< id_to_index(p2.id()) << " " + //<< p1.is_ghost() << " " + //<< p2.is_ghost() << " " + //<< p1.id() << " " + //<< p2.id() << std::endl; + //<< p1.pos() << " " + //<< p2.pos() << "\n"; }; - cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); - // verlet_list.get_max_counts(); + cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); + // verlet_list.get_max_counts(); } } else { - //if (not rebuild) { - // Else use the saved verlet list + // if (not rebuild) { + // Else use the saved verlet list verlet_list = saved_data.get_verlet_list(); } #ifdef CALIPER @@ -537,129 +540,96 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); #endif if (1) { - //ListType v_verlet_list; + // ListType v_verlet_list; /* - verlet_list = create_verlet_list( + verlet_list = create_verlet_list( max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, first_neighbor_kernel, cell_structure); - */ + */ verlet_list = ListType(0, number_of_unique_particles, max_counts); - auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); + auto const &cells = + std::as_const(cell_structure).decomposition().local_cells(); auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - - auto kernel_each = [&cells, &distance_function, - &verlet_criterion, &id_to_index, &verlet_list, - max_id, &first_neighbor_kernel] (int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { - auto &p1 = *it; - if (p1.id() > max_id) continue; - /* Pairs in this cell */ - for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { - if ((*jt).id() > max_id) continue; - if (verlet_criterion(p1, *jt, - distance_function(p1, *jt))) { - int ii = id_to_index(p1.id()); - int jj = id_to_index((*jt).id()); - verlet_list.addNeighborNonAtomic(ii, jj); - //first_neighbor_kernel(ii, jj); - } - } - } - }; - - auto kernel_neighbor = [&cells, &distance_function, - &verlet_criterion, &id_to_index, &verlet_list, - max_id, &first_neighbor_kernel] (int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { - auto &p1 = *it; - if (p1.id() > max_id) continue; - /* Pairs with neighbors */ - for (auto &neighbor : cells[i]->neighbors().red()) { - for (auto &p2 : neighbor->particles()) { - if (p2.id() > max_id) continue; - if (verlet_criterion(p1, p2, - distance_function(p1, p2))) { - int ii = id_to_index(p1.id()); - int jj = id_to_index(p2.id()); - verlet_list.addNeighbor(ii, jj); - //first_neighbor_kernel(ii, jj); - } - } - } - } - }; - - Kokkos::parallel_for("each", cells.size(), kernel_each); - Kokkos::fence(); - - Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); - Kokkos::fence(); + std::as_const(cell_structure).decomposition().box()}; + + auto kernel_each = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id, + &first_neighbor_kernel](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto &p1 = *it; + if (p1.id() > max_id) + continue; + /* Pairs in this cell */ + for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { + if ((*jt).id() > max_id) + continue; + if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { + int ii = id_to_index(p1.id()); + int jj = id_to_index((*jt).id()); + verlet_list.addNeighborNonAtomic(ii, jj); + // first_neighbor_kernel(ii, jj); + } + } + } + }; + + auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id, + &first_neighbor_kernel](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto &p1 = *it; + if (p1.id() > max_id) + continue; + /* Pairs with neighbors */ + for (auto &neighbor : cells[i]->neighbors().red()) { + for (auto &p2 : neighbor->particles()) { + if (p2.id() > max_id) + continue; + if (verlet_criterion(p1, p2, distance_function(p1, p2))) { + int ii = id_to_index(p1.id()); + int jj = id_to_index(p2.id()); + verlet_list.addNeighbor(ii, jj); + // first_neighbor_kernel(ii, jj); + } + } + } + } + }; + + Kokkos::parallel_for("each", cells.size(), kernel_each); + Kokkos::fence(); + + Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); + Kokkos::fence(); // verlet_list.get_max_counts(); } #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List by Cabana"); #endif - } //else { + } // else { { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); #endif - //using neighbor_list = Cabana::NeighborList; - //std::vector> interaction_pairs; - //std::vector> interaction_pairs; + // using neighbor_list = Cabana::NeighborList; + // std::vector> interaction_pairs; + // std::vector> interaction_pairs; /* for (int i = 0; i < number_of_unique_particles; ++i) { - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - //first_neighbor_kernel(i, j); - interaction_pairs.emplace_back(i, j); - //interaction_pairs.emplace_back(unique_particles.at(i), - // unique_particles.at(j)); - } + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + //first_neighbor_kernel(i, j); + interaction_pairs.emplace_back(i, j); + //interaction_pairs.emplace_back(unique_particles.at(i), + // unique_particles.at(j)); + } } */ // verlet_list.get_max_counts(); - /* - // Essentially same as legacy ESPRESSO - for (int i = 0; i < number_of_unique_particles; ++i) { - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - //first_neighbor_kernel(i, j); - auto thread_id = omp_get_thread_num(); - - IA_parameters const &ia_params = - nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); - //nonbonded_ias.get_ia_param(p1->type(), p2->type()); - - ParticleForce pf{}; - - Utils::Vector3d const d = box_geo.get_mi_vector( - aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), - aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); - - auto const dist = d.norm(); - - auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); - - bool do_nonbonded_flag = true; - - add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, - do_nonbonded_flag, coulomb_kernel); - - local_force(thread_id, i, 0) += pf.f[0]; - local_force(thread_id, i, 1) += pf.f[1]; - local_force(thread_id, i, 2) += pf.f[2]; - - auto opf = calc_opposing_force(pf, d); - local_force(thread_id, j, 0) += opf.f[0]; - local_force(thread_id, j, 1) += opf.f[1]; - local_force(thread_id, j, 2) += opf.f[2]; - } - } - */ // Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, @@ -669,20 +639,20 @@ void cabana_short_range( /* using neighbor_list = Cabana::NeighborList; using SimdPolicy = Cabana::SimdPolicy; - SimdPolicy simd_policy(0, (number_of_unique_particles - 1 + vector_length) / vector_length); - Cabana::simd_parallel_for(simd_policy, - [&number_of_unique_particles, &verlet_list, - &first_neighbor_kernel] (const int s, const int a) { - int i = s * vector_length + a; - if (i > number_of_unique_particles) return; - - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - first_neighbor_kernel(i, j); - } - }); - */ - + SimdPolicy simd_policy(0, (number_of_unique_particles - 1 + vector_length) + / vector_length); Cabana::simd_parallel_for(simd_policy, + [&number_of_unique_particles, &verlet_list, + &first_neighbor_kernel] (const int s, const int a) { + int i = s * vector_length + a; + if (i > number_of_unique_particles) return; + + for (int n = 0; n < + neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = + neighbor_list::getNeighbor(verlet_list, i, n); first_neighbor_kernel(i, + j); + } + }); + */ Kokkos::fence(); #ifdef CALIPER From 289721352bdd35f6592eb4c605d6dd8147e7f605 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 14 Jul 2025 20:34:14 +0200 Subject: [PATCH 53/94] Fixed warning --- src/core/aosoa_pack.hpp | 8 ++--- src/core/custom_verlet_list.hpp | 19 ++++++++--- src/core/short_range_cabana.hpp | 59 +++++++++++++++++++-------------- 3 files changed, 54 insertions(+), 32 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index d745bb82492..37afb09f793 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -24,7 +24,7 @@ #include const int vector_length = 1; -using data_types = Cabana::MemberTypes; +using data_types = Cabana::MemberTypes; //, bool>; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using AoSoA_type = Cabana::AoSoA; @@ -36,7 +36,7 @@ struct AoSoA_pack { AoSoA_type::member_slice_type<1> charge; AoSoA_type::member_slice_type<2> id; AoSoA_type::member_slice_type<3> type; - AoSoA_type::member_slice_type<4> ghost; + //AoSoA_type::member_slice_type<4> ghost; AoSoA_pack() = default; @@ -44,7 +44,7 @@ struct AoSoA_pack { : // position(Cabana::slice<0>(aosoa)), force(Cabana::slice<1>(aosoa)), // torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), - id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)), - ghost(Cabana::slice<4>(aosoa)) {} + id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)) {} + //ghost(Cabana::slice<4>(aosoa)) {} }; #endif diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 4d1155845fa..a6c263b72b3 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -75,11 +75,13 @@ class CustomVerletList nid = tmp; } count = Kokkos::atomic_fetch_add(&counts(pid), 1); +#ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } +#endif neighbors(pid, count) = nid; } @@ -95,11 +97,13 @@ class CustomVerletList nid = tmp; count = counts(pid); } +#ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } +#endif neighbors(pid, count) = nid; counts(pid) += 1; } @@ -109,14 +113,21 @@ class CustomVerletList std::size_t get_max_counts() { std::size_t max_counts = 0; std::size_t ave_counts = 0; + std::size_t ave_sq_counts = 0; for (int pid = 0; pid < counts.extent(0); ++pid) { - if (max_counts < counts(pid)) - max_counts = counts(pid); - ave_counts += counts(pid); + std::size_t count = counts(pid); + if (max_counts < count) + max_counts = count; + ave_counts += count; + ave_sq_counts += count * count; } if (counts.extent(0) != 0) { + ave_counts /= counts.extent(0); + ave_sq_counts /= counts.extent(0); + ave_sq_counts -= ave_counts * ave_counts; std::cout << "max:" << max_counts - << " ave:" << ave_counts / counts.extent(0) << std::endl; + << " ave:" << ave_counts + << " var:" << ave_sq_counts << std::endl; } return max_counts; } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 502ce56d082..0ad82508793 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -54,7 +54,7 @@ inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); - aosoa.ghost(id) = p.is_ghost(); + //aosoa.ghost(id) = p.is_ghost(); auto const pos = p.pos(); double wpos[3] = {}; for (int d = 0; d < 3; ++d) { @@ -197,17 +197,18 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Allocation"); #endif - Kokkos::View id_to_index("id_to_index", max_id + 1); Kokkos::View local_force( "local_force", num_threads, number_of_unique_particles, 3); +#ifdef ROTATION Kokkos::View local_torque( "local_torque", num_threads, number_of_unique_particles, 3); - +#endif +#ifdef NPT Kokkos::View local_virial("local_virial", num_threads, 3); - +#endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); particle_storage.resize(number_of_unique_particles); @@ -270,9 +271,9 @@ void cabana_short_range( defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) std::vector unique_particles; #endif - [[maybe_unused]] const BondedInteractionsMap &bonded_ias; - const InteractionsNonBonded &nonbonded_ias; - const BoxGeometry &box_geo; + [[maybe_unused]] const BondedInteractionsMap bonded_ias; + const InteractionsNonBonded nonbonded_ias; + const BoxGeometry box_geo; const AoSoA_pack aosoa; Kokkos::View local_force; #ifdef ROTATION @@ -307,7 +308,8 @@ void cabana_short_range( #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, const AoSoA_pack &aosoa_, + const BoxGeometry &box_geo_, + const AoSoA_pack &aosoa_, Kokkos::View local_force_, #ifdef ROTATION Kokkos::View local_torque_, @@ -329,7 +331,7 @@ void cabana_short_range( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, const Thermostat::Thermostat &thermostat_, #endif - int &num_threads_, int &mpi_rank_, int &particle_number_) + int num_threads_, int mpi_rank_, int particle_number_) : // cell(cell_), #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) @@ -366,7 +368,7 @@ void cabana_short_range( IA_parameters const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); - + ParticleForce pf{}; #ifdef NPT Utils::Vector3d virial{}; @@ -513,7 +515,7 @@ void cabana_short_range( CALI_MARK_END("Cabana - Verlet List by ESPRESSO"); #endif - FirstNeighborKernel first_neighbor_kernel( + FirstNeighborKernel first_neighbor_kernel_o( #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) unique_particles, @@ -535,12 +537,13 @@ void cabana_short_range( #endif num_threads, rank, number_of_unique_particles); + const auto& first_neighbor_kernel = first_neighbor_kernel_o; + if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); #endif if (1) { - // ListType v_verlet_list; /* verlet_list = create_verlet_list( max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, @@ -553,8 +556,9 @@ void cabana_short_range( std::as_const(cell_structure).decomposition().box()}; auto kernel_each = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id, - &first_neighbor_kernel](int i) { + &id_to_index, &verlet_list, max_id](int i) { + //&id_to_index, &verlet_list, max_id, + //&first_neighbor_kernel](int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { @@ -576,8 +580,9 @@ void cabana_short_range( }; auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id, - &first_neighbor_kernel](int i) { + &id_to_index, &verlet_list, max_id](int i) { + //&id_to_index, &verlet_list, max_id, + //&first_neighbor_kernel](int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { @@ -639,17 +644,15 @@ void cabana_short_range( /* using neighbor_list = Cabana::NeighborList; using SimdPolicy = Cabana::SimdPolicy; - SimdPolicy simd_policy(0, (number_of_unique_particles - 1 + vector_length) - / vector_length); Cabana::simd_parallel_for(simd_policy, + SimdPolicy simd_policy(0, number_of_unique_particles); + Cabana::simd_parallel_for(simd_policy, [&number_of_unique_particles, &verlet_list, &first_neighbor_kernel] (const int s, const int a) { int i = s * vector_length + a; if (i > number_of_unique_particles) return; - - for (int n = 0; n < - neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = - neighbor_list::getNeighbor(verlet_list, i, n); first_neighbor_kernel(i, - j); + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + first_neighbor_kernel(i, j); } }); */ @@ -674,21 +677,29 @@ void cabana_short_range( // Force and Torque reduction Kokkos::RangePolicy policy(0, particle_storage.size()); Kokkos::parallel_for("reduction", policy, - [&local_force, &local_torque, &unique_particles, + [&local_force, +#ifdef ROTATION + &local_torque, +#endif + &unique_particles, num_threads](const int i) { double fx = 0.; double fy = 0.; double fz = 0.; +#ifdef ROTATION double tx = 0.; double ty = 0.; double tz = 0.; +#endif for (int tid = 0; tid < num_threads; ++tid) { fx += local_force(tid, i, 0); fy += local_force(tid, i, 1); fz += local_force(tid, i, 2); +#ifdef ROTATION tx += local_torque(tid, i, 0); ty += local_torque(tid, i, 1); tz += local_torque(tid, i, 2); +#endif } auto &p = unique_particles.at(i); // auto p = From ce581ab68cfdda22628765ee68ba3af5982c8c03 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 14 Jul 2025 20:36:40 +0200 Subject: [PATCH 54/94] Formatting --- src/core/aosoa_pack.hpp | 4 ++-- src/core/custom_verlet_list.hpp | 5 ++--- src/core/short_range_cabana.hpp | 29 ++++++++++++++--------------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index 37afb09f793..ffb2ea8af25 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -36,7 +36,7 @@ struct AoSoA_pack { AoSoA_type::member_slice_type<1> charge; AoSoA_type::member_slice_type<2> id; AoSoA_type::member_slice_type<3> type; - //AoSoA_type::member_slice_type<4> ghost; + // AoSoA_type::member_slice_type<4> ghost; AoSoA_pack() = default; @@ -45,6 +45,6 @@ struct AoSoA_pack { // torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)) {} - //ghost(Cabana::slice<4>(aosoa)) {} + // ghost(Cabana::slice<4>(aosoa)) {} }; #endif diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index a6c263b72b3..9ef998dc207 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -125,9 +125,8 @@ class CustomVerletList ave_counts /= counts.extent(0); ave_sq_counts /= counts.extent(0); ave_sq_counts -= ave_counts * ave_counts; - std::cout << "max:" << max_counts - << " ave:" << ave_counts - << " var:" << ave_sq_counts << std::endl; + std::cout << "max:" << max_counts << " ave:" << ave_counts + << " var:" << ave_sq_counts << std::endl; } return max_counts; } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 0ad82508793..47cef8d7403 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -54,7 +54,7 @@ inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); - //aosoa.ghost(id) = p.is_ghost(); + // aosoa.ghost(id) = p.is_ghost(); auto const pos = p.pos(); double wpos[3] = {}; for (int d = 0; d < 3; ++d) { @@ -308,8 +308,7 @@ void cabana_short_range( #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, - const AoSoA_pack &aosoa_, + const BoxGeometry &box_geo_, const AoSoA_pack &aosoa_, Kokkos::View local_force_, #ifdef ROTATION Kokkos::View local_torque_, @@ -368,7 +367,7 @@ void cabana_short_range( IA_parameters const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); - + ParticleForce pf{}; #ifdef NPT Utils::Vector3d virial{}; @@ -537,7 +536,7 @@ void cabana_short_range( #endif num_threads, rank, number_of_unique_particles); - const auto& first_neighbor_kernel = first_neighbor_kernel_o; + const auto &first_neighbor_kernel = first_neighbor_kernel_o; if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory #ifdef CALIPER @@ -557,8 +556,8 @@ void cabana_short_range( auto kernel_each = [&cells, &distance_function, &verlet_criterion, &id_to_index, &verlet_list, max_id](int i) { - //&id_to_index, &verlet_list, max_id, - //&first_neighbor_kernel](int i) { + //&id_to_index, &verlet_list, max_id, + //&first_neighbor_kernel](int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { @@ -581,8 +580,8 @@ void cabana_short_range( auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, &id_to_index, &verlet_list, max_id](int i) { - //&id_to_index, &verlet_list, max_id, - //&first_neighbor_kernel](int i) { + //&id_to_index, &verlet_list, max_id, + //&first_neighbor_kernel](int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { @@ -650,9 +649,10 @@ void cabana_short_range( &first_neighbor_kernel] (const int s, const int a) { int i = s * vector_length + a; if (i > number_of_unique_particles) return; - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - first_neighbor_kernel(i, j); + for (int n = 0; n < + neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = + neighbor_list::getNeighbor(verlet_list, i, n); first_neighbor_kernel(i, + j); } }); */ @@ -679,10 +679,9 @@ void cabana_short_range( Kokkos::parallel_for("reduction", policy, [&local_force, #ifdef ROTATION - &local_torque, + &local_torque, #endif - &unique_particles, - num_threads](const int i) { + &unique_particles, num_threads](const int i) { double fx = 0.; double fy = 0.; double fz = 0.; From 5758065469cf233fa49ef33c0d581cabed910431 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 15 Jul 2025 15:15:47 +0200 Subject: [PATCH 55/94] Fixed a bug --- src/core/cell_system/CellStructure.hpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 39d5547bebb..b66538176aa 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -43,11 +43,9 @@ #include #include -#include #include #include #include -#include #include #include #include @@ -57,10 +55,6 @@ #include #include -#ifdef CALIPER -#include -#endif - // forward declaration to not have to import cabana #ifdef SHARED_MEMORY_PARALLELISM class CabanaData; @@ -725,9 +719,6 @@ struct CellStructure : public System::Leaf { * the pair kernel, and the verlet list is rebuilt as * we go. */ if (m_rebuild_verlet_list) { -#ifdef CALIPER - CALI_MARK_BEGIN("link_cell"); -#endif m_verlet_list.clear(); link_cell([&](Particle &p1, Particle &p2, Distance const &d) { @@ -739,13 +730,7 @@ struct CellStructure : public System::Leaf { m_rebuild_verlet_list = false; m_rebuild_cabana_verlet_list = true; -#ifdef CALIPER - CALI_MARK_END("link_cell"); -#endif } else { -#ifdef CALIPER - CALI_MARK_BEGIN("pair_kernel"); -#endif auto const maybe_box = decomposition().minimum_image_distance(); /* In this case the pair kernel is just run over the verlet list. */ if (maybe_box) { @@ -762,9 +747,6 @@ struct CellStructure : public System::Leaf { distance_function(*pair.first, *pair.second)); } } -#ifdef CALIPER - CALI_MARK_END("pair_kernel"); -#endif } } From 251f9936995e7c92f961deb4351ae53e6396be59 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 17 Jul 2025 20:36:17 +0200 Subject: [PATCH 56/94] Improved computation rate --- src/core/custom_verlet_list.hpp | 2 +- src/core/short_range_cabana.hpp | 478 +++++++++++++++----------------- 2 files changed, 227 insertions(+), 253 deletions(-) diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 9ef998dc207..c43657a4408 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -57,8 +57,8 @@ class CustomVerletList const std::size_t max_neigh) { // const std::size_t thread_number) { counts = Kokkos::View("num_neighbors", num_particles); + // neighbors = Kokkos::View( neighbors = Kokkos::View( - // neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, max_neigh); } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 47cef8d7403..a354dc79c21 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -33,7 +33,6 @@ #include "aosoa_pack.hpp" #include "cabana_data.hpp" #include "custom_verlet_list.hpp" -// #include "verlet_list_loop.hpp" #include #include #include @@ -49,14 +48,17 @@ inline double wrap(double x, double L) { return result; } -inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, - Utils::Vector3d const &box_l) { +inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa) { + //Utils::Vector3d const &box_l) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); // aosoa.ghost(id) = p.is_ghost(); auto const pos = p.pos(); - double wpos[3] = {}; + for (int d = 0; d < 3; ++d) { + aosoa.position(id, d) = pos[d]; + } + /*double wpos[3] = {}; for (int d = 0; d < 3; ++d) { // aosoa.position(id, d) = // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; @@ -64,7 +66,7 @@ inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa, } for (int d = 0; d < 3; ++d) { aosoa.position(id, d) = wpos[d]; - } + }*/ // assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); // assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); // assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); @@ -106,35 +108,19 @@ void cabana_short_range( // Cabana short range loop if (pair_cutoff > 0.) { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); // =================================================== - // Setup Cabana Variables + // Count unique particles and create Index map // =================================================== #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Setup"); + CALI_MARK_BEGIN("Cabana - Index map"); #endif - - using ListAlgorithm = Cabana::HalfNeighborTag; - using ListType = Cabana::CustomVerletList; + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Number of threads int num_threads = execution_space().concurrency(); -#ifdef CALIPER - CALI_MARK_END("Cabana - Setup"); -#endif - - // =================================================== - // Count unique particles and create Index map - // =================================================== -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Index map"); -#endif - std::unordered_set registered_index{}; std::vector unique_particles; - // std::vector sequential_particles; int index = 0; int max_id = 0; @@ -147,122 +133,77 @@ void cabana_short_range( CabanaData saved_data; - // Load saved data if we do not have to rebuild - if (!rebuild) { - saved_data = cell_structure.get_cabana_data(); - } - - // If we have to rebuild, we need to count the particles and create a new - // map + // If we have to rebuild, we need to count the particles if (rebuild) { + std::unordered_set registered_index{}; + //std::bitset<1000000> registered_index; for (auto &p : particles) { - if (cell_structure.get_local_particle(p.id())) { - if (p.id() > max_id) - max_id = p.id(); - registered_index.insert(p.id()); - unique_particles.emplace_back(&p); - // sequential_particles.emplace_back(p); - index++; - } + if (p.id() > max_id) + max_id = p.id(); + //registered_index.set(p.id()); + unique_particles.emplace_back(&p); + index++; } for (auto &p : ghost_particles) { - if (not registered_index.contains(p.id())) { - if (cell_structure.get_local_particle(p.id())) { - if (p.id() > max_id) - max_id = p.id(); - registered_index.insert(p.id()); - unique_particles.emplace_back(&p); - // sequential_particles.emplace_back(p); - index++; - } - } + if (not cell_structure.get_local_particle(p.id())) { + continue; + } + if (not cell_structure.get_local_particle(p.id())->is_ghost()) { + continue; + } + if (registered_index.contains(p.id())) { + continue; + } + //if (registered_index.test(p.id())) { + // continue; + //} + if (p.id() > max_id) + max_id = p.id(); + registered_index.insert(p.id()); + //registered_index.set(p.id()); + unique_particles.emplace_back(&p); + index++; } + registered_index.clear(); } else { // If we do not rebuild we can use the saved map + saved_data = cell_structure.get_cabana_data(); index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); max_id = saved_data.get_max_id(); } int number_of_unique_particles = index; -#ifdef CALIPER - CALI_MARK_END("Cabana - Index map"); -#endif // =================================================== - // Create and fill particle storage + // Create essential variable for MD // =================================================== -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Allocation"); -#endif - Kokkos::View id_to_index("id_to_index", max_id + 1); - Kokkos::View local_force( - "local_force", num_threads, number_of_unique_particles, 3); + Kokkos::View local_force( + "local_force", number_of_unique_particles, num_threads); #ifdef ROTATION - Kokkos::View local_torque( - "local_torque", num_threads, number_of_unique_particles, 3); + Kokkos::View local_torque( + "local_torque", number_of_unique_particles, num_threads); #endif #ifdef NPT - Kokkos::View local_virial("local_virial", - num_threads, 3); + Kokkos::View local_virial("local_virial", + num_threads); #endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); particle_storage.resize(number_of_unique_particles); // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); - auto box_l = box_geo.length(); - using policy_type = Kokkos::RangePolicy; - Kokkos::parallel_for( - "AoSoA write", policy_type(0, particle_storage.size()), - //[&unique_particles, &aosoa, &box_l](const int p_id) { - [&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); - id_to_index(unique_particles.at(p_id)->id()) = p_id; - }); - Kokkos::fence(); - // After ONLY JUST creating LinkedCellList, force calculation became slower, - // even if it is not used and It is explicitly deleted. - if (0) { - // Cabana::LinkedCellList cell_list; - double grid_min[3] = {0.0, 0.0, 0.0}; - double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; - double grid_delta[3] = {}; - int cell_num[3] = {}; - double eff_cutoff; - for (int d = 0; d < 3; ++d) { - eff_cutoff = pair_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); - } - auto *cell_list = new Cabana::LinkedCellList( - aosoa.position, grid_delta, grid_min, grid_max); - // Now permute the AoSoA (i.e. reorder the data) - Cabana::permute(*cell_list, particle_storage); - unique_particles.clear(); - for (int i = 0; i < aosoa.id.size(); ++i) { - id_to_index(aosoa.id(i)) = i; - unique_particles.emplace_back( - cell_structure.get_local_particle(aosoa.id(i))); - } - delete cell_list; - Kokkos::fence(); - /*Kokkos::parallel_for("AoSoA write", policy_type(0, - particle_storage.size()), - [&unique_particles, &aosoa, &box_l](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, - aosoa, box_l); - });*/ - } + using ListAlgorithm = Cabana::HalfNeighborTag; + using ListType = Cabana::CustomVerletList; + ListType verlet_list; #ifdef CALIPER - CALI_MARK_END("Cabana - Allocation"); + CALI_MARK_END("Cabana - Index map"); #endif // The kernel of calculate force @@ -274,13 +215,12 @@ void cabana_short_range( [[maybe_unused]] const BondedInteractionsMap bonded_ias; const InteractionsNonBonded nonbonded_ias; const BoxGeometry box_geo; - const AoSoA_pack aosoa; - Kokkos::View local_force; + Kokkos::View local_force; #ifdef ROTATION - Kokkos::View local_torque; + Kokkos::View local_torque; #endif #ifdef NPT - Kokkos::View local_virial; + Kokkos::View local_virial; #endif #ifdef COLLISION_DETECTION // std::shared_ptr @@ -295,10 +235,10 @@ void cabana_short_range( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; const Thermostat::Thermostat &thermostat; #endif - - int num_threads; - int mpi_rank; - int particle_number; + //int num_threads; + //int mpi_rank; + //int particle_number; + const AoSoA_pack aosoa; FirstNeighborKernel( // const CellStructure *cell_, @@ -308,13 +248,13 @@ void cabana_short_range( #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, const AoSoA_pack &aosoa_, - Kokkos::View local_force_, + const BoxGeometry &box_geo_, + Kokkos::View local_force_, #ifdef ROTATION - Kokkos::View local_torque_, + Kokkos::View local_torque_, #endif #ifdef NPT - Kokkos::View local_virial_, + Kokkos::View local_virial_, #endif #ifdef COLLISION_DETECTION // std::shared_ptr @@ -330,14 +270,15 @@ void cabana_short_range( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, const Thermostat::Thermostat &thermostat_, #endif - int num_threads_, int mpi_rank_, int particle_number_) + const AoSoA_pack &aosoa_) + //int num_threads_), int mpi_rank_, int particle_number_) : // cell(cell_), #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) unique_particles(unique_particles_), #endif bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - box_geo(box_geo_), aosoa(aosoa_), local_force(local_force_), + box_geo(box_geo_), local_force(local_force_), #ifdef ROTATION local_torque(local_torque_), #endif @@ -353,9 +294,7 @@ void cabana_short_range( dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), #endif - num_threads(num_threads_), mpi_rank(mpi_rank_), - particle_number(particle_number_) { - } + aosoa(aosoa_) {} KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { @@ -419,23 +358,23 @@ void cabana_short_range( coulomb_u_kernel); #endif // ETC // - local_force(thread_id, i, 0) += pf.f[0]; - local_force(thread_id, i, 1) += pf.f[1]; - local_force(thread_id, i, 2) += pf.f[2]; + local_force(i, thread_id, 0) += pf.f[0]; + local_force(i, thread_id, 1) += pf.f[1]; + local_force(i, thread_id, 2) += pf.f[2]; #ifdef ROTATION - local_torque(thread_id, i, 0) += pf.torque[0]; - local_torque(thread_id, i, 1) += pf.torque[1]; - local_torque(thread_id, i, 2) += pf.torque[2]; + local_torque(i, thread_id, 0) += pf.torque[0]; + local_torque(i, thread_id, 1) += pf.torque[1]; + local_torque(i, thread_id, 2) += pf.torque[2]; #endif auto opf = calc_opposing_force(pf, d); - local_force(thread_id, j, 0) += opf.f[0]; - local_force(thread_id, j, 1) += opf.f[1]; - local_force(thread_id, j, 2) += opf.f[2]; + local_force(j, thread_id, 0) += opf.f[0]; + local_force(j, thread_id, 1) += opf.f[1]; + local_force(j, thread_id, 2) += opf.f[2]; #ifdef ROTATION - local_torque(thread_id, j, 0) += opf.torque[0]; - local_torque(thread_id, j, 1) += opf.torque[1]; - local_torque(thread_id, j, 2) += opf.torque[2]; + local_torque(j, thread_id, 0) += opf.torque[0]; + local_torque(j, thread_id, 1) += opf.torque[1]; + local_torque(j, thread_id, 2) += opf.torque[2]; #endif #ifdef NPT local_virial(thread_id, 0) += virial[0]; @@ -451,103 +390,109 @@ void cabana_short_range( }; }; - // START VERLET_LIST - // =================================================== - // Get Verlet Pairs and Fill list - // =================================================== + // Fill the essential variable for MD + { #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List by ESPRESSO"); -#endif - ListType verlet_list; - - // Rebuild verlet list if needed - bool at_steepest_descent = cell_structure.get_steepest_descent_flag(); - int max_counts; - double max_cutoff = pair_cutoff; // system.get_interaction_range(); - if (std::isinf(max_cutoff)) { - max_counts = number_of_unique_particles; - } else { - int max_prefactor; - if (at_steepest_descent) { - max_prefactor = 8; - } else { - max_prefactor = 5; - } - max_counts = static_cast( - std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); - } - int threshold_num = 8; -#ifdef COLLISION_DETECTION - threshold_num = 64; -#endif - if (max_counts < threshold_num) { - max_counts = std::min(threshold_num, number_of_unique_particles); - } - // std::cout << "max_counts:" << max_counts << " " << max_cutoff << - // std::endl; - if (rebuild) { // Legacy Velert List + CALI_MARK_BEGIN("Cabana - Allocation"); +#endif + // =================================================== + // Fill particle storage + // =================================================== + + Kokkos::View id_to_index( + Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); + //Kokkos::view_alloc("id_to_index", Kokkos::WithoutInitializing), max_id + 1); + Kokkos::deep_copy(id_to_index, -1); + + auto box_l = box_geo.length(); + + using policy_type = Kokkos::RangePolicy; + Kokkos::parallel_for( + "AoSoA write", policy_type(0, particle_storage.size()), + //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { + [&unique_particles, &aosoa, &id_to_index](const int p_id) { + //write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); + write_particle(*unique_particles.at(p_id), p_id, aosoa); + id_to_index(unique_particles.at(p_id)->id()) = p_id; + }); + Kokkos::fence(); + // After ONLY JUST creating LinkedCellList, force calculation became slower, + // even if it is not used and It is explicitly deleted. if (0) { - verlet_list = ListType(0, number_of_unique_particles, max_counts); - auto kernel = [&verlet_list, &id_to_index](Particle const &p1, - Particle const &p2) { - verlet_list.addNeighbor(id_to_index(p1.id()), id_to_index(p2.id())); - // std::cout << "WITHSMP " - //<< id_to_index(p1.id()) << " " - //<< id_to_index(p2.id()) << " " - //<< p1.is_ghost() << " " - //<< p2.is_ghost() << " " - //<< p1.id() << " " - //<< p2.id() << std::endl; - //<< p1.pos() << " " - //<< p2.pos() << "\n"; - }; - - cell_structure.cabana_verlet_list_loop(kernel, verlet_criterion); - // verlet_list.get_max_counts(); + // Cabana::LinkedCellList cell_list; + double grid_min[3] = {0.0, 0.0, 0.0}; + double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; + double grid_delta[3] = {}; + int cell_num[3] = {}; + double eff_cutoff; + for (int d = 0; d < 3; ++d) { + eff_cutoff = pair_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + } + auto *cell_list = new Cabana::LinkedCellList( + aosoa.position, grid_delta, grid_min, grid_max); + // Now permute the AoSoA (i.e. reorder the data) + Cabana::permute(*cell_list, particle_storage); + unique_particles.clear(); + for (int i = 0; i < aosoa.id.size(); ++i) { + id_to_index(aosoa.id(i)) = i; + unique_particles.emplace_back( + cell_structure.get_local_particle(aosoa.id(i))); + } + delete cell_list; + Kokkos::fence(); + /*Kokkos::parallel_for("AoSoA write", policy_type(0, + particle_storage.size()), + [&unique_particles, &aosoa, &box_l](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, + aosoa, box_l); + });*/ } - } else { - // if (not rebuild) { - // Else use the saved verlet list - verlet_list = saved_data.get_verlet_list(); - } -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List by ESPRESSO"); -#endif - FirstNeighborKernel first_neighbor_kernel_o( -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles, -#endif - bonded_ias, nonbonded_ias, box_geo, aosoa, local_force, -#ifdef ROTATION - local_torque, -#endif -#ifdef NPT - local_virial, -#endif -#ifdef COLLISION_DETECTION - *collision_detection, -#endif - coulomb_kernel, -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, +#ifdef CALIPER + CALI_MARK_END("Cabana - Allocation"); #endif - num_threads, rank, number_of_unique_particles); - const auto &first_neighbor_kernel = first_neighbor_kernel_o; + // =================================================== + // Get Verlet Pairs and Fill list + // =================================================== - if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory + // Rebuild verlet list if needed + double max_cutoff = pair_cutoff; // system.get_interaction_range(); + if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List by Cabana"); + CALI_MARK_BEGIN("Cabana - Verlet List"); #endif - if (1) { /* verlet_list = create_verlet_list( max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, first_neighbor_kernel, cell_structure); */ + bool at_steepest_descent = cell_structure.get_steepest_descent_flag(); + int max_counts; + if (std::isinf(max_cutoff)) { + max_counts = number_of_unique_particles; + } else { + int max_prefactor; + if (at_steepest_descent) { + max_prefactor = 8; + } else { + max_prefactor = 5; + } + max_counts = static_cast( + std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); + } + int threshold_num = 8; +#ifdef COLLISION_DETECTION + threshold_num = 64; +#endif + if (max_counts < threshold_num) { + max_counts = std::min(threshold_num, number_of_unique_particles); + } + // std::cout << "max_counts:" << max_counts << " " << max_cutoff << std::endl; verlet_list = ListType(0, number_of_unique_particles, max_counts); auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); @@ -556,22 +501,23 @@ void cabana_short_range( auto kernel_each = [&cells, &distance_function, &verlet_criterion, &id_to_index, &verlet_list, max_id](int i) { - //&id_to_index, &verlet_list, max_id, - //&first_neighbor_kernel](int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto &p1 = *it; if (p1.id() > max_id) continue; + int ii = id_to_index(p1.id()); + if (ii < 0) continue; /* Pairs in this cell */ for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { if ((*jt).id() > max_id) continue; if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { - int ii = id_to_index(p1.id()); int jj = id_to_index((*jt).id()); - verlet_list.addNeighborNonAtomic(ii, jj); + if (jj >= 0) { + verlet_list.addNeighborNonAtomic(ii, jj); + } // first_neighbor_kernel(ii, jj); } } @@ -580,23 +526,24 @@ void cabana_short_range( auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, &id_to_index, &verlet_list, max_id](int i) { - //&id_to_index, &verlet_list, max_id, - //&first_neighbor_kernel](int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { - auto &p1 = *it; + auto const &p1 = *it; if (p1.id() > max_id) continue; + int ii = id_to_index(p1.id()); + if (ii < 0) continue; /* Pairs with neighbors */ for (auto &neighbor : cells[i]->neighbors().red()) { - for (auto &p2 : neighbor->particles()) { + for (auto const &p2 : neighbor->particles()) { if (p2.id() > max_id) continue; if (verlet_criterion(p1, p2, distance_function(p1, p2))) { - int ii = id_to_index(p1.id()); int jj = id_to_index(p2.id()); - verlet_list.addNeighbor(ii, jj); + if (jj >= 0) { + verlet_list.addNeighbor(ii, jj); + } // first_neighbor_kernel(ii, jj); } } @@ -609,30 +556,65 @@ void cabana_short_range( Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); Kokkos::fence(); + // verlet_list.get_max_counts(); - } + + // Save data for next iteration if we just rebuilt + CabanaData new_data(verlet_list, unique_particles, max_id); + cell_structure.set_cabana_data(std::make_unique(new_data)); #ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List by Cabana"); + CALI_MARK_END("Cabana - Verlet List"); #endif + } else if (not rebuild) { + // Else use the saved verlet list + verlet_list = saved_data.get_verlet_list(); + } } // else { { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); #endif - // using neighbor_list = Cabana::NeighborList; + FirstNeighborKernel first_neighbor_kernel_o( +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles, +#endif + bonded_ias, nonbonded_ias, box_geo, local_force, +#ifdef ROTATION + local_torque, +#endif +#ifdef NPT + local_virial, +#endif +#ifdef COLLISION_DETECTION + *collision_detection, +#endif + coulomb_kernel, +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, +#endif + aosoa);//num_threads, rank, number_of_unique_particles); + + const auto &first_neighbor_kernel = first_neighbor_kernel_o; // std::vector> interaction_pairs; // std::vector> interaction_pairs; - /* + /*using neighbor_list = Cabana::NeighborList; + if (rank == 3) { for (int i = 0; i < number_of_unique_particles; ++i) { + std::cout << "i:" << i; for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = neighbor_list::getNeighbor(verlet_list, i, n); + std::cout << " " << j; //first_neighbor_kernel(i, j); - interaction_pairs.emplace_back(i, j); + //interaction_pairs.emplace_back(i, j); //interaction_pairs.emplace_back(unique_particles.at(i), // unique_particles.at(j)); } + std::cout << std::endl; } - */ + }*/ + // // verlet_list.get_max_counts(); // Kokkos::RangePolicy policy(0, particle_storage.size()); @@ -663,14 +645,6 @@ void cabana_short_range( #endif } - // Save data for next iteration if we just rebuilt - if (rebuild) { - // CabanaData new_data(verlet_list, unique_particles, - // unique_particles.size()); - CabanaData new_data(verlet_list, unique_particles, max_id); - cell_structure.set_cabana_data(std::make_unique(new_data)); - } - #ifdef CALIPER CALI_MARK_BEGIN("Cabana - reduction Forces"); #endif @@ -691,13 +665,13 @@ void cabana_short_range( double tz = 0.; #endif for (int tid = 0; tid < num_threads; ++tid) { - fx += local_force(tid, i, 0); - fy += local_force(tid, i, 1); - fz += local_force(tid, i, 2); + fx += local_force(i, tid, 0); + fy += local_force(i, tid, 1); + fz += local_force(i, tid, 2); #ifdef ROTATION - tx += local_torque(tid, i, 0); - ty += local_torque(tid, i, 1); - tz += local_torque(tid, i, 2); + tx += local_torque(i, tid, 0); + ty += local_torque(i, tid, 1); + tz += local_torque(i, tid, 2); #endif } auto &p = unique_particles.at(i); From 7d0b60e9f4ce6098c871c022792051a57b0cc8ae Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 17 Jul 2025 20:37:21 +0200 Subject: [PATCH 57/94] Formatting --- src/core/short_range_cabana.hpp | 241 ++++++++++++++++---------------- 1 file changed, 123 insertions(+), 118 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index a354dc79c21..3e990e9f208 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -48,8 +48,9 @@ inline double wrap(double x, double L) { return result; } -inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa) { - //Utils::Vector3d const &box_l) { +inline void write_particle(Particle const &p, int const &id, + AoSoA_pack &aosoa) { + // Utils::Vector3d const &box_l) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); @@ -137,34 +138,34 @@ void cabana_short_range( if (rebuild) { std::unordered_set registered_index{}; - //std::bitset<1000000> registered_index; + // std::bitset<1000000> registered_index; for (auto &p : particles) { - if (p.id() > max_id) - max_id = p.id(); - //registered_index.set(p.id()); - unique_particles.emplace_back(&p); - index++; + if (p.id() > max_id) + max_id = p.id(); + // registered_index.set(p.id()); + unique_particles.emplace_back(&p); + index++; } for (auto &p : ghost_particles) { if (not cell_structure.get_local_particle(p.id())) { - continue; - } - if (not cell_structure.get_local_particle(p.id())->is_ghost()) { - continue; - } + continue; + } + if (not cell_structure.get_local_particle(p.id())->is_ghost()) { + continue; + } if (registered_index.contains(p.id())) { - continue; - } - //if (registered_index.test(p.id())) { - // continue; - //} - if (p.id() > max_id) - max_id = p.id(); - registered_index.insert(p.id()); - //registered_index.set(p.id()); - unique_particles.emplace_back(&p); - index++; + continue; + } + // if (registered_index.test(p.id())) { + // continue; + // } + if (p.id() > max_id) + max_id = p.id(); + registered_index.insert(p.id()); + // registered_index.set(p.id()); + unique_particles.emplace_back(&p); + index++; } registered_index.clear(); } else { @@ -189,7 +190,7 @@ void cabana_short_range( #endif #ifdef NPT Kokkos::View local_virial("local_virial", - num_threads); + num_threads); #endif Cabana::AoSoA particle_storage( "particles", number_of_unique_particles); @@ -235,9 +236,9 @@ void cabana_short_range( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; const Thermostat::Thermostat &thermostat; #endif - //int num_threads; - //int mpi_rank; - //int particle_number; + // int num_threads; + // int mpi_rank; + // int particle_number; const AoSoA_pack aosoa; FirstNeighborKernel( @@ -248,8 +249,7 @@ void cabana_short_range( #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, - Kokkos::View local_force_, + const BoxGeometry &box_geo_, Kokkos::View local_force_, #ifdef ROTATION Kokkos::View local_torque_, #endif @@ -270,8 +270,8 @@ void cabana_short_range( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, const Thermostat::Thermostat &thermostat_, #endif - const AoSoA_pack &aosoa_) - //int num_threads_), int mpi_rank_, int particle_number_) + const AoSoA_pack &aosoa_) + // int num_threads_), int mpi_rank_, int particle_number_) : // cell(cell_), #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) @@ -294,7 +294,8 @@ void cabana_short_range( dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), #endif - aosoa(aosoa_) {} + aosoa(aosoa_) { + } KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { @@ -400,56 +401,57 @@ void cabana_short_range( // =================================================== Kokkos::View id_to_index( - Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); - //Kokkos::view_alloc("id_to_index", Kokkos::WithoutInitializing), max_id + 1); + Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); + // Kokkos::view_alloc("id_to_index", Kokkos::WithoutInitializing), max_id + // + 1); Kokkos::deep_copy(id_to_index, -1); auto box_l = box_geo.length(); using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for( - "AoSoA write", policy_type(0, particle_storage.size()), - //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { - [&unique_particles, &aosoa, &id_to_index](const int p_id) { - //write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); - write_particle(*unique_particles.at(p_id), p_id, aosoa); - id_to_index(unique_particles.at(p_id)->id()) = p_id; - }); + "AoSoA write", policy_type(0, particle_storage.size()), + //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { + [&unique_particles, &aosoa, &id_to_index](const int p_id) { + // write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); + write_particle(*unique_particles.at(p_id), p_id, aosoa); + id_to_index(unique_particles.at(p_id)->id()) = p_id; + }); Kokkos::fence(); - // After ONLY JUST creating LinkedCellList, force calculation became slower, - // even if it is not used and It is explicitly deleted. + // After ONLY JUST creating LinkedCellList, force calculation became + // slower, even if it is not used and It is explicitly deleted. if (0) { - // Cabana::LinkedCellList cell_list; - double grid_min[3] = {0.0, 0.0, 0.0}; - double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; - double grid_delta[3] = {}; - int cell_num[3] = {}; - double eff_cutoff; - for (int d = 0; d < 3; ++d) { - eff_cutoff = pair_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); - } - auto *cell_list = new Cabana::LinkedCellList( - aosoa.position, grid_delta, grid_min, grid_max); - // Now permute the AoSoA (i.e. reorder the data) - Cabana::permute(*cell_list, particle_storage); - unique_particles.clear(); - for (int i = 0; i < aosoa.id.size(); ++i) { - id_to_index(aosoa.id(i)) = i; - unique_particles.emplace_back( - cell_structure.get_local_particle(aosoa.id(i))); - } - delete cell_list; - Kokkos::fence(); - /*Kokkos::parallel_for("AoSoA write", policy_type(0, - particle_storage.size()), - [&unique_particles, &aosoa, &box_l](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, - aosoa, box_l); - });*/ + // Cabana::LinkedCellList cell_list; + double grid_min[3] = {0.0, 0.0, 0.0}; + double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; + double grid_delta[3] = {}; + int cell_num[3] = {}; + double eff_cutoff; + for (int d = 0; d < 3; ++d) { + eff_cutoff = pair_cutoff; + if (eff_cutoff > box_l[d]) + eff_cutoff = box_l[d]; + cell_num[d] = static_cast(box_l[d] / eff_cutoff); + grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); + } + auto *cell_list = new Cabana::LinkedCellList( + aosoa.position, grid_delta, grid_min, grid_max); + // Now permute the AoSoA (i.e. reorder the data) + Cabana::permute(*cell_list, particle_storage); + unique_particles.clear(); + for (int i = 0; i < aosoa.id.size(); ++i) { + id_to_index(aosoa.id(i)) = i; + unique_particles.emplace_back( + cell_structure.get_local_particle(aosoa.id(i))); + } + delete cell_list; + Kokkos::fence(); + /*Kokkos::parallel_for("AoSoA write", policy_type(0, + particle_storage.size()), + [&unique_particles, &aosoa, &box_l](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, + aosoa, box_l); + });*/ } #ifdef CALIPER @@ -471,28 +473,29 @@ void cabana_short_range( max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, first_neighbor_kernel, cell_structure); */ - bool at_steepest_descent = cell_structure.get_steepest_descent_flag(); - int max_counts; - if (std::isinf(max_cutoff)) { - max_counts = number_of_unique_particles; - } else { - int max_prefactor; - if (at_steepest_descent) { - max_prefactor = 8; - } else { - max_prefactor = 5; - } - max_counts = static_cast( - std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); - } - int threshold_num = 8; + bool at_steepest_descent = cell_structure.get_steepest_descent_flag(); + int max_counts; + if (std::isinf(max_cutoff)) { + max_counts = number_of_unique_particles; + } else { + int max_prefactor; + if (at_steepest_descent) { + max_prefactor = 8; + } else { + max_prefactor = 5; + } + max_counts = static_cast( + std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); + } + int threshold_num = 8; #ifdef COLLISION_DETECTION - threshold_num = 64; + threshold_num = 64; #endif - if (max_counts < threshold_num) { - max_counts = std::min(threshold_num, number_of_unique_particles); - } - // std::cout << "max_counts:" << max_counts << " " << max_cutoff << std::endl; + if (max_counts < threshold_num) { + max_counts = std::min(threshold_num, number_of_unique_particles); + } + // std::cout << "max_counts:" << max_counts << " " << max_cutoff << + // std::endl; verlet_list = ListType(0, number_of_unique_particles, max_counts); auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); @@ -508,16 +511,17 @@ void cabana_short_range( if (p1.id() > max_id) continue; int ii = id_to_index(p1.id()); - if (ii < 0) continue; + if (ii < 0) + continue; /* Pairs in this cell */ for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { if ((*jt).id() > max_id) continue; if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { int jj = id_to_index((*jt).id()); - if (jj >= 0) { + if (jj >= 0) { verlet_list.addNeighborNonAtomic(ii, jj); - } + } // first_neighbor_kernel(ii, jj); } } @@ -533,7 +537,8 @@ void cabana_short_range( if (p1.id() > max_id) continue; int ii = id_to_index(p1.id()); - if (ii < 0) continue; + if (ii < 0) + continue; /* Pairs with neighbors */ for (auto &neighbor : cells[i]->neighbors().red()) { for (auto const &p2 : neighbor->particles()) { @@ -541,9 +546,9 @@ void cabana_short_range( continue; if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); - if (jj >= 0) { + if (jj >= 0) { verlet_list.addNeighbor(ii, jj); - } + } // first_neighbor_kernel(ii, jj); } } @@ -556,18 +561,18 @@ void cabana_short_range( Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); Kokkos::fence(); - + // verlet_list.get_max_counts(); - + // Save data for next iteration if we just rebuilt - CabanaData new_data(verlet_list, unique_particles, max_id); - cell_structure.set_cabana_data(std::make_unique(new_data)); + CabanaData new_data(verlet_list, unique_particles, max_id); + cell_structure.set_cabana_data(std::make_unique(new_data)); #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List"); #endif } else if (not rebuild) { - // Else use the saved verlet list - verlet_list = saved_data.get_verlet_list(); + // Else use the saved verlet list + verlet_list = saved_data.get_verlet_list(); } } // else { { @@ -577,24 +582,24 @@ void cabana_short_range( FirstNeighborKernel first_neighbor_kernel_o( #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles, + unique_particles, #endif - bonded_ias, nonbonded_ias, box_geo, local_force, + bonded_ias, nonbonded_ias, box_geo, local_force, #ifdef ROTATION - local_torque, + local_torque, #endif #ifdef NPT - local_virial, + local_virial, #endif #ifdef COLLISION_DETECTION - *collision_detection, + *collision_detection, #endif - coulomb_kernel, + coulomb_kernel, #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, + dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, #endif - aosoa);//num_threads, rank, number_of_unique_particles); + aosoa); // num_threads, rank, number_of_unique_particles); const auto &first_neighbor_kernel = first_neighbor_kernel_o; // std::vector> interaction_pairs; @@ -602,16 +607,16 @@ void cabana_short_range( /*using neighbor_list = Cabana::NeighborList; if (rank == 3) { for (int i = 0; i < number_of_unique_particles; ++i) { - std::cout << "i:" << i; + std::cout << "i:" << i; for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = neighbor_list::getNeighbor(verlet_list, i, n); - std::cout << " " << j; + std::cout << " " << j; //first_neighbor_kernel(i, j); //interaction_pairs.emplace_back(i, j); //interaction_pairs.emplace_back(unique_particles.at(i), // unique_particles.at(j)); } - std::cout << std::endl; + std::cout << std::endl; } }*/ // From 592f4073111b657dd0bc9759006e9ebcba29f0fe Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 18 Jul 2025 20:40:16 +0200 Subject: [PATCH 58/94] Implemented practical max_counts --- src/core/cell_system/CellStructure.cpp | 1 + src/core/cell_system/CellStructure.hpp | 4 ++ src/core/custom_verlet_list.hpp | 18 ++++- src/core/short_range_cabana.hpp | 92 +++++++++++++++----------- 4 files changed, 75 insertions(+), 40 deletions(-) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 7bca218b819..ebd832a3b00 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -319,6 +319,7 @@ void CellStructure::set_verlet_skin(double value) { m_verlet_skin = value; m_verlet_skin_set = true; m_rebuild_cabana_verlet_list = true; + max_counts = -1; get_system().on_verlet_skin_change(); } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index b66538176aa..948dc45f0b1 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -662,6 +662,7 @@ struct CellStructure : public System::Leaf { private: std::unique_ptr m_cabana_data; bool steepest_descent_flag = true; + std::size_t max_counts = -1; public: void set_cabana_data(std::unique_ptr data); @@ -678,6 +679,9 @@ struct CellStructure : public System::Leaf { void set_steepest_descent_flag(bool flag) { steepest_descent_flag = flag; } bool get_steepest_descent_flag() { return steepest_descent_flag; } + void set_max_counts(std::size_t value) { max_counts = value; } + std::size_t get_max_counts() { return max_counts; } + template void cabana_link_cell(Kernel kernel) { auto const local_cells_span = decomposition().local_cells(); auto const first = boost::make_indirect_iterator(local_cells_span.begin()); diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index c43657a4408..4820054e084 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -110,7 +110,7 @@ class CustomVerletList // Find max counts KOKKOS_INLINE_FUNCTION - std::size_t get_max_counts() { + std::size_t get_variance_max_counts() { std::size_t max_counts = 0; std::size_t ave_counts = 0; std::size_t ave_sq_counts = 0; @@ -130,6 +130,22 @@ class CustomVerletList } return max_counts; } + + KOKKOS_INLINE_FUNCTION + std::size_t get_max_counts() { + int max; + Kokkos::Max max_reduce( max ); + Kokkos::parallel_reduce( + "custom_velet_list::reduce_max", + Kokkos::RangePolicy(0, counts.size()), + [&]( const int i, int& value ) { + if ( counts( i ) > value ) + value = counts( i ); + }, + max_reduce ); + Kokkos::fence(); + return static_cast( max ); + } }; template diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 3e990e9f208..c63b8fabe7e 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -50,24 +50,31 @@ inline double wrap(double x, double L) { inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa) { - // Utils::Vector3d const &box_l) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); - // aosoa.ghost(id) = p.is_ghost(); auto const pos = p.pos(); for (int d = 0; d < 3; ++d) { aosoa.position(id, d) = pos[d]; } - /*double wpos[3] = {}; +} + +inline void write_particle_permute(Particle const &p, int const &id, + AoSoA_pack &aosoa, Utils::Vector3d const &box_l) { + aosoa.id(id) = p.id(); + aosoa.charge(id) = p.q(); + aosoa.type(id) = p.type(); + // aosoa.ghost(id) = p.is_ghost(); + auto const pos = p.pos(); + double wpos[3] = {}; for (int d = 0; d < 3; ++d) { - // aosoa.position(id, d) = + //aosoa.position(id, d) = // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; wpos[d] = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; } for (int d = 0; d < 3; ++d) { aosoa.position(id, d) = wpos[d]; - }*/ + } // assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); // assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); // assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); @@ -402,25 +409,25 @@ void cabana_short_range( Kokkos::View id_to_index( Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); - // Kokkos::view_alloc("id_to_index", Kokkos::WithoutInitializing), max_id - // + 1); Kokkos::deep_copy(id_to_index, -1); - auto box_l = box_geo.length(); + //auto box_l = box_geo.length(); using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for( "AoSoA write", policy_type(0, particle_storage.size()), //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { [&unique_particles, &aosoa, &id_to_index](const int p_id) { - // write_particle(*unique_particles.at(p_id), p_id, aosoa, box_l); + //write_particle_permute(*unique_particles.at(p_id), p_id, aosoa, box_l); write_particle(*unique_particles.at(p_id), p_id, aosoa); id_to_index(unique_particles.at(p_id)->id()) = p_id; }); Kokkos::fence(); + // After ONLY JUST creating LinkedCellList, force calculation became // slower, even if it is not used and It is explicitly deleted. if (0) { + auto box_l = box_geo.length(); // Cabana::LinkedCellList cell_list; double grid_min[3] = {0.0, 0.0, 0.0}; double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; @@ -446,12 +453,12 @@ void cabana_short_range( } delete cell_list; Kokkos::fence(); - /*Kokkos::parallel_for("AoSoA write", policy_type(0, + Kokkos::parallel_for("touch aosoa", policy_type(0, particle_storage.size()), - [&unique_particles, &aosoa, &box_l](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, - aosoa, box_l); - });*/ + [&aosoa](const int p_id) { + volatile double tmp = aosoa.position(p_id, 0); + (void)tmp; + }); } #ifdef CALIPER @@ -463,11 +470,12 @@ void cabana_short_range( // =================================================== // Rebuild verlet list if needed - double max_cutoff = pair_cutoff; // system.get_interaction_range(); - if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory + // if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory + if (rebuild) { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List"); #endif + double max_cutoff = pair_cutoff; /* verlet_list = create_verlet_list( max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, @@ -475,27 +483,32 @@ void cabana_short_range( */ bool at_steepest_descent = cell_structure.get_steepest_descent_flag(); int max_counts; - if (std::isinf(max_cutoff)) { - max_counts = number_of_unique_particles; - } else { - int max_prefactor; - if (at_steepest_descent) { - max_prefactor = 8; - } else { - max_prefactor = 5; - } - max_counts = static_cast( + int practical_max = cell_structure.get_max_counts(); + if (not std::isinf(max_cutoff)) { + if (practical_max > 0) { + max_counts = practical_max + 1; + } else { + int max_prefactor; + if (at_steepest_descent) { + max_prefactor = 8; + } else { + max_prefactor = 5; + } + max_counts = static_cast( std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); - } - int threshold_num = 8; + int threshold_num = 8; #ifdef COLLISION_DETECTION - threshold_num = 64; + threshold_num = 64; #endif - if (max_counts < threshold_num) { - max_counts = std::min(threshold_num, number_of_unique_particles); - } - // std::cout << "max_counts:" << max_counts << " " << max_cutoff << - // std::endl; + if (max_counts < threshold_num) { + max_counts = std::min(threshold_num, number_of_unique_particles); + } + } + } else { + max_counts = number_of_unique_particles; + } + //std::cout << "max_counts:" << max_counts << " " + // << max_cutoff << std::endl; verlet_list = ListType(0, number_of_unique_particles, max_counts); auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); @@ -562,7 +575,9 @@ void cabana_short_range( Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); Kokkos::fence(); - // verlet_list.get_max_counts(); + if (practical_max < 0) { + cell_structure.set_max_counts(verlet_list.get_max_counts()); + } // Save data for next iteration if we just rebuilt CabanaData new_data(verlet_list, unique_particles, max_id); @@ -636,10 +651,9 @@ void cabana_short_range( &first_neighbor_kernel] (const int s, const int a) { int i = s * vector_length + a; if (i > number_of_unique_particles) return; - for (int n = 0; n < - neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = - neighbor_list::getNeighbor(verlet_list, i, n); first_neighbor_kernel(i, - j); + for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { + int j = neighbor_list::getNeighbor(verlet_list, i, n); + first_neighbor_kernel(i,j); } }); */ From 1b8fb3575c6f341155dadc34d7714649c91fb258 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 21 Jul 2025 20:36:28 +0200 Subject: [PATCH 59/94] Fixed bugs --- src/core/aosoa_pack.hpp | 2 +- src/core/cell_system/CellStructure.cpp | 2 + src/core/cell_system/CellStructure.hpp | 10 +++- src/core/custom_verlet_list.hpp | 28 ++++----- src/core/integrate.cpp | 6 +- src/core/short_range_cabana.hpp | 80 ++++++++++++-------------- 6 files changed, 65 insertions(+), 63 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index ffb2ea8af25..c7d85e84104 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -24,7 +24,7 @@ #include const int vector_length = 1; -using data_types = Cabana::MemberTypes; //, bool>; +using data_types = Cabana::MemberTypes; //, bool>; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; using AoSoA_type = Cabana::AoSoA; diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index ebd832a3b00..96026ca6cd6 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -319,7 +319,9 @@ void CellStructure::set_verlet_skin(double value) { m_verlet_skin = value; m_verlet_skin_set = true; m_rebuild_cabana_verlet_list = true; +#ifdef SHARED_MEMORY_PARALLELISM max_counts = -1; +#endif get_system().on_verlet_skin_change(); } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 948dc45f0b1..beef57b5ade 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -661,7 +661,8 @@ struct CellStructure : public System::Leaf { #ifdef SHARED_MEMORY_PARALLELISM private: std::unique_ptr m_cabana_data; - bool steepest_descent_flag = true; + // bool steepest_descent_flag = true; + std::size_t max_prefactor = 8; std::size_t max_counts = -1; public: @@ -676,8 +677,11 @@ struct CellStructure : public System::Leaf { return m_rebuild_cabana_verlet_list; } - void set_steepest_descent_flag(bool flag) { steepest_descent_flag = flag; } - bool get_steepest_descent_flag() { return steepest_descent_flag; } + // void set_steepest_descent_flag(bool flag) { steepest_descent_flag = flag; } + // bool get_steepest_descent_flag() { return steepest_descent_flag; } + + void set_max_prefactor(std::size_t value) { max_prefactor = value; } + std::size_t get_max_prefactor() { return max_prefactor; } void set_max_counts(std::size_t value) { max_counts = value; } std::size_t get_max_counts() { return max_counts; } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 4820054e084..299ae851933 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -70,18 +70,19 @@ class CustomVerletList std::size_t count_n = counts(nid); if (count > count_n) { + //if (pid > nid) { int tmp = pid; pid = nid; nid = tmp; } count = Kokkos::atomic_fetch_add(&counts(pid), 1); -#ifndef NDEBUG +//#ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } -#endif +//#endif neighbors(pid, count) = nid; } @@ -92,18 +93,19 @@ class CustomVerletList std::size_t count_n = counts(nid); if (count > count_n) { + //if (pid > nid) { int tmp = pid; pid = nid; nid = tmp; count = counts(pid); } -#ifndef NDEBUG +//#ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } -#endif +//#endif neighbors(pid, count) = nid; counts(pid) += 1; } @@ -134,17 +136,17 @@ class CustomVerletList KOKKOS_INLINE_FUNCTION std::size_t get_max_counts() { int max; - Kokkos::Max max_reduce( max ); + Kokkos::Max max_reduce(max); Kokkos::parallel_reduce( - "custom_velet_list::reduce_max", - Kokkos::RangePolicy(0, counts.size()), - [&]( const int i, int& value ) { - if ( counts( i ) > value ) - value = counts( i ); - }, - max_reduce ); + "custom_velet_list::reduce_max", + Kokkos::RangePolicy(0, counts.size()), + [&](const int i, int &value) { + if (counts(i) > value) + value = counts(i); + }, + max_reduce); Kokkos::fence(); - return static_cast( max ); + return static_cast(max); } }; diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index e3c372344f3..9f99114508e 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -526,12 +526,14 @@ int System::System::integrate(int n_steps, int reuse_forces) { lb_active = lb.is_solver_set(); ek_active = ek.is_ready_for_propagation(); #ifdef SHARED_MEMORY_PARALLELISM - cell_structure->set_steepest_descent_flag(false); + //cell_structure->set_steepest_descent_flag(false); + cell_structure->set_max_prefactor(8); #endif } #ifdef SHARED_MEMORY_PARALLELISM else { - cell_structure->set_steepest_descent_flag(true); + //cell_structure->set_steepest_descent_flag(true); + cell_structure->set_max_prefactor(5); } #endif auto const calc_md_steps_per_tau = [this](double tau) { diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index c63b8fabe7e..5c5402b76fa 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -60,7 +60,8 @@ inline void write_particle(Particle const &p, int const &id, } inline void write_particle_permute(Particle const &p, int const &id, - AoSoA_pack &aosoa, Utils::Vector3d const &box_l) { + AoSoA_pack &aosoa, + Utils::Vector3d const &box_l) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); @@ -68,8 +69,8 @@ inline void write_particle_permute(Particle const &p, int const &id, auto const pos = p.pos(); double wpos[3] = {}; for (int d = 0; d < 3; ++d) { - //aosoa.position(id, d) = - // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; + // aosoa.position(id, d) = + // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; wpos[d] = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; } for (int d = 0; d < 3; ++d) { @@ -411,14 +412,15 @@ void cabana_short_range( Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); Kokkos::deep_copy(id_to_index, -1); - //auto box_l = box_geo.length(); + // auto box_l = box_geo.length(); //permute using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for( "AoSoA write", policy_type(0, particle_storage.size()), - //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) { + //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) {//permute [&unique_particles, &aosoa, &id_to_index](const int p_id) { - //write_particle_permute(*unique_particles.at(p_id), p_id, aosoa, box_l); + //write_particle_permute(*unique_particles.at(p_id), p_id, aosoa, + // box_l);//permute write_particle(*unique_particles.at(p_id), p_id, aosoa); id_to_index(unique_particles.at(p_id)->id()) = p_id; }); @@ -453,12 +455,12 @@ void cabana_short_range( } delete cell_list; Kokkos::fence(); - Kokkos::parallel_for("touch aosoa", policy_type(0, - particle_storage.size()), - [&aosoa](const int p_id) { - volatile double tmp = aosoa.position(p_id, 0); - (void)tmp; - }); + /*Kokkos::parallel_for("touch aosoa", + policy_type(0, particle_storage.size()), + [&aosoa](const int p_id) { + volatile double tmp = aosoa.position(p_id, 0); + (void)tmp; + });*/ } #ifdef CALIPER @@ -470,32 +472,24 @@ void cabana_short_range( // =================================================== // Rebuild verlet list if needed - // if (rebuild and max_cutoff != INACTIVE_CUTOFF) { // Shared memory + // if (rebuild and pair_cutoff != INACTIVE_CUTOFF) { // Shared memory if (rebuild) { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List"); #endif - double max_cutoff = pair_cutoff; /* verlet_list = create_verlet_list( - max_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, + pair_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, first_neighbor_kernel, cell_structure); */ - bool at_steepest_descent = cell_structure.get_steepest_descent_flag(); int max_counts; - int practical_max = cell_structure.get_max_counts(); - if (not std::isinf(max_cutoff)) { - if (practical_max > 0) { - max_counts = practical_max + 1; - } else { - int max_prefactor; - if (at_steepest_descent) { - max_prefactor = 8; - } else { - max_prefactor = 5; - } - max_counts = static_cast( - std::ceil(max_prefactor * max_cutoff * max_cutoff * max_cutoff)); + int practical_max = cell_structure.get_max_counts(); + if (not std::isinf(pair_cutoff)) { + if (practical_max > 0) { + max_counts = practical_max + 2; + } else { + max_counts = static_cast(std::ceil(cell_structure.get_max_prefactor() + * pair_cutoff * pair_cutoff * pair_cutoff)); int threshold_num = 8; #ifdef COLLISION_DETECTION threshold_num = 64; @@ -503,12 +497,12 @@ void cabana_short_range( if (max_counts < threshold_num) { max_counts = std::min(threshold_num, number_of_unique_particles); } - } - } else { + } + } else { max_counts = number_of_unique_particles; - } - //std::cout << "max_counts:" << max_counts << " " - // << max_cutoff << std::endl; + } + // std::cout << "max_counts:" << max_counts << " " + // << pair_cutoff << std::endl; verlet_list = ListType(0, number_of_unique_particles, max_counts); auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); @@ -575,9 +569,7 @@ void cabana_short_range( Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); Kokkos::fence(); - if (practical_max < 0) { - cell_structure.set_max_counts(verlet_list.get_max_counts()); - } + cell_structure.set_max_counts(verlet_list.get_max_counts()); // Save data for next iteration if we just rebuilt CabanaData new_data(verlet_list, unique_particles, max_id); @@ -594,7 +586,7 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); #endif - FirstNeighborKernel first_neighbor_kernel_o( + FirstNeighborKernel first_neighbor_kernel( #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) unique_particles, @@ -616,11 +608,11 @@ void cabana_short_range( #endif aosoa); // num_threads, rank, number_of_unique_particles); - const auto &first_neighbor_kernel = first_neighbor_kernel_o; + const auto &kernel_force = first_neighbor_kernel; // std::vector> interaction_pairs; // std::vector> interaction_pairs; /*using neighbor_list = Cabana::NeighborList; - if (rank == 3) { + if (rank == 0) { for (int i = 0; i < number_of_unique_particles; ++i) { std::cout << "i:" << i; for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { @@ -638,7 +630,7 @@ void cabana_short_range( // verlet_list.get_max_counts(); // Kokkos::RangePolicy policy(0, particle_storage.size()); - Cabana::neighbor_parallel_for(policy, first_neighbor_kernel, verlet_list, + Cabana::neighbor_parallel_for(policy, kernel_force, verlet_list, Cabana::FirstNeighborsTag(), Cabana::TeamOpTag()); // @@ -651,9 +643,9 @@ void cabana_short_range( &first_neighbor_kernel] (const int s, const int a) { int i = s * vector_length + a; if (i > number_of_unique_particles) return; - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - first_neighbor_kernel(i,j); + for (int n = 0; n < + neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = + neighbor_list::getNeighbor(verlet_list, i, n); first_neighbor_kernel(i,j); } }); */ From 97be7d73e515abe2ad9a95ac3d811fea889e72a9 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 21 Jul 2025 20:38:32 +0200 Subject: [PATCH 60/94] Formatting --- src/core/custom_verlet_list.hpp | 12 ++++++------ src/core/integrate.cpp | 4 ++-- src/core/short_range_cabana.hpp | 12 +++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 299ae851933..dc8fc710941 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -70,19 +70,19 @@ class CustomVerletList std::size_t count_n = counts(nid); if (count > count_n) { - //if (pid > nid) { + // if (pid > nid) { int tmp = pid; pid = nid; nid = tmp; } count = Kokkos::atomic_fetch_add(&counts(pid), 1); -//#ifndef NDEBUG + // #ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } -//#endif + // #endif neighbors(pid, count) = nid; } @@ -93,19 +93,19 @@ class CustomVerletList std::size_t count_n = counts(nid); if (count > count_n) { - //if (pid > nid) { + // if (pid > nid) { int tmp = pid; pid = nid; nid = tmp; count = counts(pid); } -//#ifndef NDEBUG + // #ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } -//#endif + // #endif neighbors(pid, count) = nid; counts(pid) += 1; } diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index 9f99114508e..930394afc57 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -526,13 +526,13 @@ int System::System::integrate(int n_steps, int reuse_forces) { lb_active = lb.is_solver_set(); ek_active = ek.is_ready_for_propagation(); #ifdef SHARED_MEMORY_PARALLELISM - //cell_structure->set_steepest_descent_flag(false); + // cell_structure->set_steepest_descent_flag(false); cell_structure->set_max_prefactor(8); #endif } #ifdef SHARED_MEMORY_PARALLELISM else { - //cell_structure->set_steepest_descent_flag(true); + // cell_structure->set_steepest_descent_flag(true); cell_structure->set_max_prefactor(5); } #endif diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 5c5402b76fa..e253a808b79 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -417,10 +417,11 @@ void cabana_short_range( using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for( "AoSoA write", policy_type(0, particle_storage.size()), - //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) {//permute + //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) + //{//permute [&unique_particles, &aosoa, &id_to_index](const int p_id) { - //write_particle_permute(*unique_particles.at(p_id), p_id, aosoa, - // box_l);//permute + // write_particle_permute(*unique_particles.at(p_id), p_id, aosoa, + // box_l);//permute write_particle(*unique_particles.at(p_id), p_id, aosoa); id_to_index(unique_particles.at(p_id)->id()) = p_id; }); @@ -488,8 +489,9 @@ void cabana_short_range( if (practical_max > 0) { max_counts = practical_max + 2; } else { - max_counts = static_cast(std::ceil(cell_structure.get_max_prefactor() - * pair_cutoff * pair_cutoff * pair_cutoff)); + max_counts = static_cast( + std::ceil(cell_structure.get_max_prefactor() * pair_cutoff * + pair_cutoff * pair_cutoff)); int threshold_num = 8; #ifdef COLLISION_DETECTION threshold_num = 64; From 932510b6e36755d9f74f1f052503b8d617d7c326 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 24 Jul 2025 17:21:34 +0200 Subject: [PATCH 61/94] Small refactoring --- src/core/short_range_cabana.hpp | 495 ++++++++++++++------------------ 1 file changed, 211 insertions(+), 284 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index e253a808b79..80ca608c263 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -22,7 +22,7 @@ #include "config/config.hpp" #include "cell_system/CellStructure.hpp" -#include "lees_edwards/lees_edwards.hpp" +//#include "lees_edwards/lees_edwards.hpp" #ifdef CALIPER #include @@ -33,13 +33,14 @@ #include "aosoa_pack.hpp" #include "cabana_data.hpp" #include "custom_verlet_list.hpp" +#include "forces_cabana.hpp" #include #include -#include -#include -#include +//#include +//#include +//#include #include -#include +//#include inline double wrap(double x, double L) { auto result = x - std::floor(x / L) * L; @@ -81,9 +82,132 @@ inline void write_particle_permute(Particle const &p, int const &id, // assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); } +inline void set_index_map(std::vector &unique_particles, + ParticleRange const &particles, + ParticleRange const &ghost_particles, + CellStructure const &cell_structure, + int &index, int &max_id) { + std::unordered_set registered_index{}; + for (auto &p : particles) { + if (p.id() > max_id) + max_id = p.id(); + unique_particles.emplace_back(&p); + index++; + } + + for (auto &p : ghost_particles) { + if (not cell_structure.get_local_particle(p.id())) { + continue; + } + if (not cell_structure.get_local_particle(p.id())->is_ghost()) { + continue; + } + if (registered_index.contains(p.id())) { + continue; + } + if (p.id() > max_id) + max_id = p.id(); + registered_index.insert(p.id()); + unique_particles.emplace_back(&p); + index++; + } + registered_index.clear(); +} + +inline int estimate_max_counts(const double pair_cutoff, const int number_of_unique_particles, + CellStructure &cell_structure) { + int max_counts; + if (not std::isinf(pair_cutoff)) { + max_counts = static_cast( + std::ceil(cell_structure.get_max_prefactor() * pair_cutoff * + pair_cutoff * pair_cutoff)); + int threshold_num = 8; +#ifdef COLLISION_DETECTION + threshold_num = 64; +#endif + if (max_counts < threshold_num) { + max_counts = std::min(threshold_num, number_of_unique_particles); + } + } else { + max_counts = number_of_unique_particles; + } + return max_counts; +} + +using ListAlgorithm = Cabana::HalfNeighborTag; +using ListType = Cabana::CustomVerletList; +template +inline void set_verlet_list(CellStructure const &cell_structure, + VerletCriterion const &verlet_criterion, Kokkos::View const &id_to_index, + ListType &verlet_list, const int max_id) { + auto const &cells = + std::as_const(cell_structure).decomposition().local_cells(); + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; + + auto kernel_each = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto &p1 = *it; + if (p1.id() > max_id) + continue; + int ii = id_to_index(p1.id()); + if (ii < 0) + continue; + /* Pairs in this cell */ + for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { + if ((*jt).id() > max_id) + continue; + if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { + int jj = id_to_index((*jt).id()); + if (jj >= 0) { + verlet_list.addNeighborNonAtomic(ii, jj); + } + } + } + } + }; + + auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto const &p1 = *it; + if (p1.id() > max_id) + continue; + int ii = id_to_index(p1.id()); + if (ii < 0) + continue; + /* Pairs with neighbors */ + for (auto &neighbor : cells[i]->neighbors().red()) { + for (auto const &p2 : neighbor->particles()) { + if (p2.id() > max_id) + continue; + if (verlet_criterion(p1, p2, distance_function(p1, p2))) { + int jj = id_to_index(p2.id()); + if (jj >= 0) { + verlet_list.addNeighbor(ii, jj); + } + } + } + } + } + }; + + Kokkos::parallel_for("each", cells.size(), kernel_each); + Kokkos::fence(); + + Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); + Kokkos::fence(); +} + template void cabana_short_range( - BondKernel bond_kernel, + BondKernel const &bond_kernel, [[maybe_unused]] BondedInteractionsMap const &bonded_ias, Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, @@ -94,8 +218,8 @@ void cabana_short_range( #endif CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, - InteractionsNonBonded &nonbonded_ias, ParticleRange particles, - ParticleRange ghost_particles, + InteractionsNonBonded const &nonbonded_ias, ParticleRange const &particles, + ParticleRange const &ghost_particles, VerletCriterion const &verlet_criterion = {}) { #ifdef CALIPER CALI_CXX_MARK_FUNCTION; @@ -104,13 +228,11 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Espresso - Bond Kernel"); #endif - assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); if (bond_cutoff >= 0.) { cell_structure.bond_loop(bond_kernel); } - #ifdef CALIPER CALI_MARK_END("Espresso - Bond Kernel"); #endif @@ -123,69 +245,32 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Index map"); #endif - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - // Number of threads int num_threads = execution_space().concurrency(); std::vector unique_particles; - int index = 0; + int number_of_unique_particles = 0; int max_id = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); - // if (rank == 0) { - // std::cout << "\nFor CABANA rebuild " << rebuild - // << " " << rank << std::endl; - // } - CabanaData saved_data; + ListType verlet_list; - // If we have to rebuild, we need to count the particles if (rebuild) { - - std::unordered_set registered_index{}; - // std::bitset<1000000> registered_index; - for (auto &p : particles) { - if (p.id() > max_id) - max_id = p.id(); - // registered_index.set(p.id()); - unique_particles.emplace_back(&p); - index++; - } - - for (auto &p : ghost_particles) { - if (not cell_structure.get_local_particle(p.id())) { - continue; - } - if (not cell_structure.get_local_particle(p.id())->is_ghost()) { - continue; - } - if (registered_index.contains(p.id())) { - continue; - } - // if (registered_index.test(p.id())) { - // continue; - // } - if (p.id() > max_id) - max_id = p.id(); - registered_index.insert(p.id()); - // registered_index.set(p.id()); - unique_particles.emplace_back(&p); - index++; - } - registered_index.clear(); + // If we have to rebuild, we need to count the particles + set_index_map(unique_particles, particles, ghost_particles, + cell_structure, number_of_unique_particles, max_id); } else { // If we do not rebuild we can use the saved map + CabanaData saved_data; saved_data = cell_structure.get_cabana_data(); - index = saved_data.get_index(); unique_particles = saved_data.get_unique_particles(); + number_of_unique_particles = saved_data.get_index(); max_id = saved_data.get_max_id(); + verlet_list = saved_data.get_verlet_list(); } - int number_of_unique_particles = index; - // =================================================== // Create essential variable for MD // =================================================== @@ -206,11 +291,6 @@ void cabana_short_range( // particle properties are defined in aosoa_pack.hpp auto aosoa = AoSoA_pack(particle_storage); - using ListAlgorithm = Cabana::HalfNeighborTag; - using ListType = Cabana::CustomVerletList; - ListType verlet_list; - #ifdef CALIPER CALI_MARK_END("Cabana - Index map"); #endif @@ -230,11 +310,6 @@ void cabana_short_range( #endif #ifdef NPT Kokkos::View local_virial; -#endif -#ifdef COLLISION_DETECTION - // std::shared_ptr - // collision_detection; - mutable CollisionDetection::CollisionDetection collision_detection; #endif Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ @@ -244,13 +319,9 @@ void cabana_short_range( Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; const Thermostat::Thermostat &thermostat; #endif - // int num_threads; - // int mpi_rank; - // int particle_number; const AoSoA_pack aosoa; FirstNeighborKernel( - // const CellStructure *cell_, #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) std::vector &unique_particles_, @@ -263,11 +334,6 @@ void cabana_short_range( #endif #ifdef NPT Kokkos::View local_virial_, -#endif -#ifdef COLLISION_DETECTION - // std::shared_ptr - // collision_detection_, - CollisionDetection::CollisionDetection collision_detection_, #endif Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ @@ -279,8 +345,7 @@ void cabana_short_range( const Thermostat::Thermostat &thermostat_, #endif const AoSoA_pack &aosoa_) - // int num_threads_), int mpi_rank_, int particle_number_) - : // cell(cell_), + : #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) unique_particles(unique_particles_), @@ -292,9 +357,6 @@ void cabana_short_range( #endif #ifdef NPT local_virial(local_virial_), -#endif -#ifdef COLLISION_DETECTION - collision_detection(collision_detection_), #endif coulomb_kernel(coulomb_kernel_), #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ @@ -309,9 +371,6 @@ void cabana_short_range( void operator()(int i, int j) const { auto thread_id = omp_get_thread_num(); - // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - // std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << - // " " << IA_parameters const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); @@ -330,11 +389,6 @@ void cabana_short_range( #ifdef EXCLUSIONS auto p1 = unique_particles.at(i); auto p2 = unique_particles.at(j); - // auto p1 = cell->get_local_particle(aosoa.id(i)); - // auto p2 = cell->get_local_particle(aosoa.id(j)); - - // if (p1 == nullptr or p2 == nullptr) - // return; bool do_nonbonded_flag = do_nonbonded(*p1, *p2); #else @@ -351,11 +405,6 @@ void cabana_short_range( #ifndef EXCLUSIONS auto p1 = unique_particles.at(i); auto p2 = unique_particles.at(j); - // auto p1 = cell->get_local_particle(aosoa.id(i)); - // auto p2 = cell->get_local_particle(aosoa.id(j)); - - // if (p1 == nullptr or p2 == nullptr) - // return; #endif // NOT EXCLUSIONS add_non_bonded_pair_force_with_p( const_cast(*p1), const_cast(*p2), pf, @@ -366,7 +415,6 @@ void cabana_short_range( box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, coulomb_u_kernel); #endif // ETC - // local_force(i, thread_id, 0) += pf.f[0]; local_force(i, thread_id, 1) += pf.f[1]; local_force(i, thread_id, 2) += pf.f[2]; @@ -390,12 +438,6 @@ void cabana_short_range( local_virial(thread_id, 1) += virial[1]; local_virial(thread_id, 2) += virial[2]; #endif - -#ifdef COLLISION_DETECTION - // if (not collision_detection.is_off()) { - // collision_detection.detect_collision(*p1, *p2, dist2); - // } -#endif }; }; @@ -412,166 +454,94 @@ void cabana_short_range( Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); Kokkos::deep_copy(id_to_index, -1); - // auto box_l = box_geo.length(); //permute - using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for( "AoSoA write", policy_type(0, particle_storage.size()), - //[&unique_particles, &aosoa, &box_l, &id_to_index](const int p_id) - //{//permute [&unique_particles, &aosoa, &id_to_index](const int p_id) { - // write_particle_permute(*unique_particles.at(p_id), p_id, aosoa, - // box_l);//permute write_particle(*unique_particles.at(p_id), p_id, aosoa); id_to_index(unique_particles.at(p_id)->id()) = p_id; }); Kokkos::fence(); - // After ONLY JUST creating LinkedCellList, force calculation became - // slower, even if it is not used and It is explicitly deleted. - if (0) { - auto box_l = box_geo.length(); - // Cabana::LinkedCellList cell_list; - double grid_min[3] = {0.0, 0.0, 0.0}; - double grid_max[3] = {box_l[0], box_l[1], box_l[2]}; - double grid_delta[3] = {}; - int cell_num[3] = {}; - double eff_cutoff; - for (int d = 0; d < 3; ++d) { - eff_cutoff = pair_cutoff; - if (eff_cutoff > box_l[d]) - eff_cutoff = box_l[d]; - cell_num[d] = static_cast(box_l[d] / eff_cutoff); - grid_delta[d] = std::nextafter(box_l[d] / cell_num[d], 0); - } - auto *cell_list = new Cabana::LinkedCellList( - aosoa.position, grid_delta, grid_min, grid_max); - // Now permute the AoSoA (i.e. reorder the data) - Cabana::permute(*cell_list, particle_storage); - unique_particles.clear(); - for (int i = 0; i < aosoa.id.size(); ++i) { - id_to_index(aosoa.id(i)) = i; - unique_particles.emplace_back( - cell_structure.get_local_particle(aosoa.id(i))); - } - delete cell_list; - Kokkos::fence(); - /*Kokkos::parallel_for("touch aosoa", - policy_type(0, particle_storage.size()), - [&aosoa](const int p_id) { - volatile double tmp = aosoa.position(p_id, 0); - (void)tmp; - });*/ - } - #ifdef CALIPER CALI_MARK_END("Cabana - Allocation"); #endif // =================================================== - // Get Verlet Pairs and Fill list + // Get Verlet Pairs and Fill Verlet list // =================================================== // Rebuild verlet list if needed - // if (rebuild and pair_cutoff != INACTIVE_CUTOFF) { // Shared memory if (rebuild) { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List"); #endif - /* - verlet_list = create_verlet_list( - pair_cutoff, max_counts, aosoa, unique_particles, verlet_criterion, - first_neighbor_kernel, cell_structure); - */ - int max_counts; - int practical_max = cell_structure.get_max_counts(); - if (not std::isinf(pair_cutoff)) { - if (practical_max > 0) { - max_counts = practical_max + 2; - } else { - max_counts = static_cast( - std::ceil(cell_structure.get_max_prefactor() * pair_cutoff * - pair_cutoff * pair_cutoff)); - int threshold_num = 8; -#ifdef COLLISION_DETECTION - threshold_num = 64; -#endif - if (max_counts < threshold_num) { - max_counts = std::min(threshold_num, number_of_unique_particles); - } - } - } else { - max_counts = number_of_unique_particles; - } - // std::cout << "max_counts:" << max_counts << " " - // << pair_cutoff << std::endl; + int max_counts = estimate_max_counts(pair_cutoff, number_of_unique_particles, cell_structure); verlet_list = ListType(0, number_of_unique_particles, max_counts); - auto const &cells = - std::as_const(cell_structure).decomposition().local_cells(); - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - - auto kernel_each = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { - auto &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs in this cell */ - for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { - if ((*jt).id() > max_id) - continue; - if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { - int jj = id_to_index((*jt).id()); - if (jj >= 0) { - verlet_list.addNeighborNonAtomic(ii, jj); - } - // first_neighbor_kernel(ii, jj); - } - } - } - }; - - auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { - auto const &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs with neighbors */ - for (auto &neighbor : cells[i]->neighbors().red()) { - for (auto const &p2 : neighbor->particles()) { - if (p2.id() > max_id) - continue; - if (verlet_criterion(p1, p2, distance_function(p1, p2))) { - int jj = id_to_index(p2.id()); - if (jj >= 0) { - verlet_list.addNeighbor(ii, jj); - } - // first_neighbor_kernel(ii, jj); - } - } - } - } - }; - - Kokkos::parallel_for("each", cells.size(), kernel_each); - Kokkos::fence(); - - Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); - Kokkos::fence(); - - cell_structure.set_max_counts(verlet_list.get_max_counts()); + + //set_verlet_list(cell_structure, verlet_criterion, id_to_index, verlet_list, max_id); + auto const &cells = + std::as_const(cell_structure).decomposition().local_cells(); + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; + + auto kernel_each = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto &p1 = *it; + if (p1.id() > max_id) + continue; + int ii = id_to_index(p1.id()); + if (ii < 0) + continue; + /* Pairs in this cell */ + for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { + if ((*jt).id() > max_id) + continue; + if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { + int jj = id_to_index((*jt).id()); + if (jj >= 0) { + verlet_list.addNeighborNonAtomic(ii, jj); + } + } + } + } + }; + + auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto const &p1 = *it; + if (p1.id() > max_id) + continue; + int ii = id_to_index(p1.id()); + if (ii < 0) + continue; + /* Pairs with neighbors */ + for (auto &neighbor : cells[i]->neighbors().red()) { + for (auto const &p2 : neighbor->particles()) { + if (p2.id() > max_id) + continue; + if (verlet_criterion(p1, p2, distance_function(p1, p2))) { + int jj = id_to_index(p2.id()); + if (jj >= 0) { + verlet_list.addNeighbor(ii, jj); + } + } + } + } + } + }; + + Kokkos::parallel_for("each", cells.size(), kernel_each); + Kokkos::fence(); + + Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); + Kokkos::fence(); // Save data for next iteration if we just rebuilt CabanaData new_data(verlet_list, unique_particles, max_id); @@ -579,16 +549,14 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List"); #endif - } else if (not rebuild) { - // Else use the saved verlet list - verlet_list = saved_data.get_verlet_list(); } - } // else { + } { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); #endif FirstNeighborKernel first_neighbor_kernel( + // ForcesKernel first_neighbor_kernel( #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) unique_particles, @@ -599,59 +567,20 @@ void cabana_short_range( #endif #ifdef NPT local_virial, -#endif -#ifdef COLLISION_DETECTION - *collision_detection, #endif coulomb_kernel, #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) or defined(NPT) dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, #endif - aosoa); // num_threads, rank, number_of_unique_particles); + aosoa); const auto &kernel_force = first_neighbor_kernel; - // std::vector> interaction_pairs; - // std::vector> interaction_pairs; - /*using neighbor_list = Cabana::NeighborList; - if (rank == 0) { - for (int i = 0; i < number_of_unique_particles; ++i) { - std::cout << "i:" << i; - for (int n = 0; n < neighbor_list::numNeighbor(verlet_list, i); ++n) { - int j = neighbor_list::getNeighbor(verlet_list, i, n); - std::cout << " " << j; - //first_neighbor_kernel(i, j); - //interaction_pairs.emplace_back(i, j); - //interaction_pairs.emplace_back(unique_particles.at(i), - // unique_particles.at(j)); - } - std::cout << std::endl; - } - }*/ - // - // verlet_list.get_max_counts(); - // + // verlet_list.get_variance_max_counts(); Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, kernel_force, verlet_list, Cabana::FirstNeighborsTag(), Cabana::TeamOpTag()); - // - /* - using neighbor_list = Cabana::NeighborList; - using SimdPolicy = Cabana::SimdPolicy; - SimdPolicy simd_policy(0, number_of_unique_particles); - Cabana::simd_parallel_for(simd_policy, - [&number_of_unique_particles, &verlet_list, - &first_neighbor_kernel] (const int s, const int a) { - int i = s * vector_length + a; - if (i > number_of_unique_particles) return; - for (int n = 0; n < - neighbor_list::numNeighbor(verlet_list, i); ++n) { int j = - neighbor_list::getNeighbor(verlet_list, i, n); first_neighbor_kernel(i,j); - } - }); - */ - Kokkos::fence(); #ifdef CALIPER CALI_MARK_END("Cabana - calc Force"); @@ -688,8 +617,6 @@ void cabana_short_range( #endif } auto &p = unique_particles.at(i); - // auto p = - // cell_structure.get_local_particle(aosoa.id(i)); p->force() += Utils::Vector3d{fx, fy, fz}; #ifdef ROTATION p->torque() += Utils::Vector3d{tx, ty, tz}; From 2a6753a9c6d083755e6d75e38de8da7a11c28900 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 24 Jul 2025 17:24:58 +0200 Subject: [PATCH 62/94] Formatting --- src/core/short_range_cabana.hpp | 222 ++++++++++++++++---------------- 1 file changed, 110 insertions(+), 112 deletions(-) diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 80ca608c263..e96ff26e9cc 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -22,7 +22,7 @@ #include "config/config.hpp" #include "cell_system/CellStructure.hpp" -//#include "lees_edwards/lees_edwards.hpp" +// #include "lees_edwards/lees_edwards.hpp" #ifdef CALIPER #include @@ -36,11 +36,8 @@ #include "forces_cabana.hpp" #include #include -//#include -//#include -//#include +#include #include -//#include inline double wrap(double x, double L) { auto result = x - std::floor(x / L) * L; @@ -66,7 +63,6 @@ inline void write_particle_permute(Particle const &p, int const &id, aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); - // aosoa.ghost(id) = p.is_ghost(); auto const pos = p.pos(); double wpos[3] = {}; for (int d = 0; d < 3; ++d) { @@ -82,11 +78,11 @@ inline void write_particle_permute(Particle const &p, int const &id, // assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); } -inline void set_index_map(std::vector &unique_particles, - ParticleRange const &particles, - ParticleRange const &ghost_particles, - CellStructure const &cell_structure, - int &index, int &max_id) { +inline void set_index_map(std::vector &unique_particles, + ParticleRange const &particles, + ParticleRange const &ghost_particles, + CellStructure const &cell_structure, int &index, + int &max_id) { std::unordered_set registered_index{}; for (auto &p : particles) { if (p.id() > max_id) @@ -114,13 +110,14 @@ inline void set_index_map(std::vector &unique_particles, registered_index.clear(); } -inline int estimate_max_counts(const double pair_cutoff, const int number_of_unique_particles, - CellStructure &cell_structure) { +inline int estimate_max_counts(const double pair_cutoff, + const int number_of_unique_particles, + CellStructure &cell_structure) { int max_counts; if (not std::isinf(pair_cutoff)) { - max_counts = static_cast( - std::ceil(cell_structure.get_max_prefactor() * pair_cutoff * - pair_cutoff * pair_cutoff)); + max_counts = + static_cast(std::ceil(cell_structure.get_max_prefactor() * + pair_cutoff * pair_cutoff * pair_cutoff)); int threshold_num = 8; #ifdef COLLISION_DETECTION threshold_num = 64; @@ -136,64 +133,63 @@ inline int estimate_max_counts(const double pair_cutoff, const int number_of_uni using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; + Cabana::VerletLayout2D>; template inline void set_verlet_list(CellStructure const &cell_structure, - VerletCriterion const &verlet_criterion, Kokkos::View const &id_to_index, - ListType &verlet_list, const int max_id) { + VerletCriterion const &verlet_criterion, + Kokkos::View const &id_to_index, + ListType &verlet_list, const int max_id) { auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; auto kernel_each = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { + &id_to_index, &verlet_list, max_id](int i) { auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { + for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto &p1 = *it; if (p1.id() > max_id) - continue; + continue; int ii = id_to_index(p1.id()); if (ii < 0) - continue; + continue; /* Pairs in this cell */ for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { - if ((*jt).id() > max_id) - continue; - if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { - int jj = id_to_index((*jt).id()); - if (jj >= 0) { - verlet_list.addNeighborNonAtomic(ii, jj); - } - } + if ((*jt).id() > max_id) + continue; + if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { + int jj = id_to_index((*jt).id()); + if (jj >= 0) { + verlet_list.addNeighborNonAtomic(ii, jj); + } + } } } }; auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { + &id_to_index, &verlet_list, max_id](int i) { auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { + for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto const &p1 = *it; if (p1.id() > max_id) - continue; + continue; int ii = id_to_index(p1.id()); if (ii < 0) - continue; + continue; /* Pairs with neighbors */ for (auto &neighbor : cells[i]->neighbors().red()) { - for (auto const &p2 : neighbor->particles()) { - if (p2.id() > max_id) - continue; - if (verlet_criterion(p1, p2, distance_function(p1, p2))) { - int jj = id_to_index(p2.id()); - if (jj >= 0) { - verlet_list.addNeighbor(ii, jj); - } - } - } + for (auto const &p2 : neighbor->particles()) { + if (p2.id() > max_id) + continue; + if (verlet_criterion(p1, p2, distance_function(p1, p2))) { + int jj = id_to_index(p2.id()); + if (jj >= 0) { + verlet_list.addNeighbor(ii, jj); + } + } + } } } }; @@ -260,7 +256,7 @@ void cabana_short_range( if (rebuild) { // If we have to rebuild, we need to count the particles set_index_map(unique_particles, particles, ghost_particles, - cell_structure, number_of_unique_particles, max_id); + cell_structure, number_of_unique_particles, max_id); } else { // If we do not rebuild we can use the saved map CabanaData saved_data; @@ -476,72 +472,74 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List"); #endif - int max_counts = estimate_max_counts(pair_cutoff, number_of_unique_particles, cell_structure); + int max_counts = estimate_max_counts( + pair_cutoff, number_of_unique_particles, cell_structure); verlet_list = ListType(0, number_of_unique_particles, max_counts); - //set_verlet_list(cell_structure, verlet_criterion, id_to_index, verlet_list, max_id); - auto const &cells = - std::as_const(cell_structure).decomposition().local_cells(); - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - - auto kernel_each = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { - auto &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs in this cell */ - for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { - if ((*jt).id() > max_id) - continue; - if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { - int jj = id_to_index((*jt).id()); - if (jj >= 0) { - verlet_list.addNeighborNonAtomic(ii, jj); - } - } - } - } - }; - - auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { - auto const &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs with neighbors */ - for (auto &neighbor : cells[i]->neighbors().red()) { - for (auto const &p2 : neighbor->particles()) { - if (p2.id() > max_id) - continue; - if (verlet_criterion(p1, p2, distance_function(p1, p2))) { - int jj = id_to_index(p2.id()); - if (jj >= 0) { - verlet_list.addNeighbor(ii, jj); - } - } - } - } - } - }; - - Kokkos::parallel_for("each", cells.size(), kernel_each); - Kokkos::fence(); - - Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); - Kokkos::fence(); + // set_verlet_list(cell_structure, verlet_criterion, id_to_index, + // verlet_list, max_id); + auto const &cells = + std::as_const(cell_structure).decomposition().local_cells(); + auto const distance_function = detail::MinimalImageDistance{ + std::as_const(cell_structure).decomposition().box()}; + + auto kernel_each = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto &p1 = *it; + if (p1.id() > max_id) + continue; + int ii = id_to_index(p1.id()); + if (ii < 0) + continue; + /* Pairs in this cell */ + for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { + if ((*jt).id() > max_id) + continue; + if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { + int jj = id_to_index((*jt).id()); + if (jj >= 0) { + verlet_list.addNeighborNonAtomic(ii, jj); + } + } + } + } + }; + + auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](int i) { + auto &local_particles = cells[i]->particles(); + for (auto it = local_particles.begin(); it != local_particles.end(); + ++it) { + auto const &p1 = *it; + if (p1.id() > max_id) + continue; + int ii = id_to_index(p1.id()); + if (ii < 0) + continue; + /* Pairs with neighbors */ + for (auto &neighbor : cells[i]->neighbors().red()) { + for (auto const &p2 : neighbor->particles()) { + if (p2.id() > max_id) + continue; + if (verlet_criterion(p1, p2, distance_function(p1, p2))) { + int jj = id_to_index(p2.id()); + if (jj >= 0) { + verlet_list.addNeighbor(ii, jj); + } + } + } + } + } + }; + + Kokkos::parallel_for("each", cells.size(), kernel_each); + Kokkos::fence(); + + Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); + Kokkos::fence(); // Save data for next iteration if we just rebuilt CabanaData new_data(verlet_list, unique_particles, max_id); From 3130c2871f2c3991fba95940563c70618035068d Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 24 Jul 2025 18:22:26 +0200 Subject: [PATCH 63/94] Implemented non-Atomic Add for Verlet List --- src/core/custom_verlet_list.hpp | 18 +++++++++++++++++- src/core/integrate.cpp | 2 +- src/core/short_range_cabana.hpp | 12 ++++++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index dc8fc710941..96627333846 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -65,7 +65,7 @@ class CustomVerletList // Method to add a neighbor KOKKOS_INLINE_FUNCTION - void addNeighbor(int pid, int nid) { + void addNeighborAtomic(int pid, int nid) { std::size_t count = counts(pid); std::size_t count_n = counts(nid); @@ -90,6 +90,22 @@ class CustomVerletList KOKKOS_INLINE_FUNCTION void addNeighborNonAtomic(int pid, int nid) { std::size_t count = counts(pid); + + // #ifndef NDEBUG + if (count >= neighbors.extent(1)) { + throw std::runtime_error( + // Kokkos::abort( + "Number of count is larger than VerletList size."); + } + // #endif + neighbors(pid, count) = nid; + counts(pid) += 1; + } + + // Non atomic and load balancing method to add a neighbor + KOKKOS_INLINE_FUNCTION + void addNeighborLoadBalancing(int pid, int nid) { + std::size_t count = counts(pid); std::size_t count_n = counts(nid); if (count > count_n) { diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index 930394afc57..46f37c77c1a 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -533,7 +533,7 @@ int System::System::integrate(int n_steps, int reuse_forces) { #ifdef SHARED_MEMORY_PARALLELISM else { // cell_structure->set_steepest_descent_flag(true); - cell_structure->set_max_prefactor(5); + cell_structure->set_max_prefactor(7);//5 } #endif auto const calc_md_steps_per_tau = [this](double tau) { diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index e96ff26e9cc..a6699a7bfd8 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -161,7 +161,8 @@ inline void set_verlet_list(CellStructure const &cell_structure, if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { int jj = id_to_index((*jt).id()); if (jj >= 0) { - verlet_list.addNeighborNonAtomic(ii, jj); + //verlet_list.addNeighborNonAtomic(ii, jj); + verlet_list.addNeighborLoadBalancing(ii, jj); } } } @@ -186,7 +187,8 @@ inline void set_verlet_list(CellStructure const &cell_structure, if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); if (jj >= 0) { - verlet_list.addNeighbor(ii, jj); + //verlet_list.addNeighbor(ii, jj); + verlet_list.addNeighborNonAtomic(ii, jj); } } } @@ -501,7 +503,8 @@ void cabana_short_range( if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { int jj = id_to_index((*jt).id()); if (jj >= 0) { - verlet_list.addNeighborNonAtomic(ii, jj); + //verlet_list.addNeighborNonAtomic(ii, jj); + verlet_list.addNeighborLoadBalancing(ii, jj); } } } @@ -527,7 +530,8 @@ void cabana_short_range( if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); if (jj >= 0) { - verlet_list.addNeighbor(ii, jj); + //verlet_list.addNeighbor(ii, jj); + verlet_list.addNeighborNonAtomic(ii, jj); } } } From 8cb8de27c5962f188644913f3174bb26db149078 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 24 Jul 2025 18:24:28 +0200 Subject: [PATCH 64/94] Formatting and add new header of functor for force calc --- src/core/forces_cabana.hpp | 192 ++++++++++++++++++++++++++++++++ src/core/integrate.cpp | 2 +- src/core/short_range_cabana.hpp | 10 +- 3 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 src/core/forces_cabana.hpp diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp new file mode 100644 index 00000000000..bc4c894a896 --- /dev/null +++ b/src/core/forces_cabana.hpp @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2010-2025 The ESPResSo project + * + * This file is part of ESPResSo. + * + * ESPResSo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ESPResSo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#ifdef CALIPER +#include +#endif + +#ifdef SHARED_MEMORY_PARALLELISM + +#include "aosoa_pack.hpp" +#include "cabana_data.hpp" +#include "custom_verlet_list.hpp" +#include +#include +#include +#include +#include +#include +#include + +struct ForcesKernel { +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ +defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + std::vector unique_particles; +#endif + [[maybe_unused]] const BondedInteractionsMap bonded_ias; + const InteractionsNonBonded nonbonded_ias; + const BoxGeometry box_geo; + Kokkos::View local_force; +#ifdef ROTATION + Kokkos::View local_torque; +#endif +#ifdef NPT + Kokkos::View local_virial; +#endif + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ +defined(DPD) or defined(DIPOLES) or defined(NPT) + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; + const Thermostat::Thermostat &thermostat; +#endif + // int num_threads; + // int mpi_rank; + // int particle_number; + const AoSoA_pack aosoa; + + ForcesKernel( + // const CellStructure *cell_, +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ +defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + std::vector &unique_particles_, +#endif + [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, + const InteractionsNonBonded &nonbonded_ias_, + const BoxGeometry &box_geo_, Kokkos::View local_force_, +#ifdef ROTATION + Kokkos::View local_torque_, +#endif +#ifdef NPT + Kokkos::View local_virial_, +#endif + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ +defined(DPD) or defined(DIPOLES) or defined(NPT) + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const + *elc_kernel_, + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, + const Thermostat::Thermostat &thermostat_, +#endif + const AoSoA_pack &aosoa_) + // int num_threads_), int mpi_rank_, int particle_number_) + : // cell(cell_), +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ +defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles(unique_particles_), +#endif + bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + box_geo(box_geo_), local_force(local_force_), +#ifdef ROTATION + local_torque(local_torque_), +#endif +#ifdef NPT + local_virial(local_virial_), +#endif + coulomb_kernel(coulomb_kernel_), +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ +defined(DPD) or defined(DIPOLES) or defined(NPT) + dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), + coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), +#endif + aosoa(aosoa_) { + } + + KOKKOS_FORCEINLINE_FUNCTION + void operator()(int i, int j) const { + + auto thread_id = omp_get_thread_num(); + // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); + // std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << + // " " << + + IA_parameters const &ia_params = + nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); + + ParticleForce pf{}; +#ifdef NPT + Utils::Vector3d virial{}; +#endif + Utils::Vector3d const d = box_geo.get_mi_vector( + aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), + aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); + auto const dist = d.norm(); + + auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); + +#ifdef EXCLUSIONS + auto p1 = unique_particles.at(i); + auto p2 = unique_particles.at(j); + + bool do_nonbonded_flag = do_nonbonded(*p1, *p2); +#else + bool do_nonbonded_flag = true; +#endif + + add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, + do_nonbonded_flag, coulomb_kernel); + +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ +defined(DPD) or defined(DIPOLES) or defined(NPT) + auto const dist2 = dist * dist; + +#ifndef EXCLUSIONS + auto p1 = unique_particles.at(i); + auto p2 = unique_particles.at(j); +#endif // NOT EXCLUSIONS + add_non_bonded_pair_force_with_p( + const_cast(*p1), const_cast(*p2), pf, +#ifdef NPT + virial, +#endif // NPT + d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, + box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, + coulomb_u_kernel); +#endif // ETC + // + local_force(i, thread_id, 0) += pf.f[0]; + local_force(i, thread_id, 1) += pf.f[1]; + local_force(i, thread_id, 2) += pf.f[2]; +#ifdef ROTATION + local_torque(i, thread_id, 0) += pf.torque[0]; + local_torque(i, thread_id, 1) += pf.torque[1]; + local_torque(i, thread_id, 2) += pf.torque[2]; +#endif + + auto opf = calc_opposing_force(pf, d); + local_force(j, thread_id, 0) += opf.f[0]; + local_force(j, thread_id, 1) += opf.f[1]; + local_force(j, thread_id, 2) += opf.f[2]; +#ifdef ROTATION + local_torque(j, thread_id, 0) += opf.torque[0]; + local_torque(j, thread_id, 1) += opf.torque[1]; + local_torque(j, thread_id, 2) += opf.torque[2]; +#endif +#ifdef NPT + local_virial(thread_id, 0) += virial[0]; + local_virial(thread_id, 1) += virial[1]; + local_virial(thread_id, 2) += virial[2]; +#endif + } +}; +#endif diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index 46f37c77c1a..55fc07933d5 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -533,7 +533,7 @@ int System::System::integrate(int n_steps, int reuse_forces) { #ifdef SHARED_MEMORY_PARALLELISM else { // cell_structure->set_steepest_descent_flag(true); - cell_structure->set_max_prefactor(7);//5 + cell_structure->set_max_prefactor(7); // 5 } #endif auto const calc_md_steps_per_tau = [this](double tau) { diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index a6699a7bfd8..3aadb35d390 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -161,7 +161,7 @@ inline void set_verlet_list(CellStructure const &cell_structure, if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { int jj = id_to_index((*jt).id()); if (jj >= 0) { - //verlet_list.addNeighborNonAtomic(ii, jj); + // verlet_list.addNeighborNonAtomic(ii, jj); verlet_list.addNeighborLoadBalancing(ii, jj); } } @@ -187,7 +187,7 @@ inline void set_verlet_list(CellStructure const &cell_structure, if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); if (jj >= 0) { - //verlet_list.addNeighbor(ii, jj); + // verlet_list.addNeighbor(ii, jj); verlet_list.addNeighborNonAtomic(ii, jj); } } @@ -503,8 +503,8 @@ void cabana_short_range( if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { int jj = id_to_index((*jt).id()); if (jj >= 0) { - //verlet_list.addNeighborNonAtomic(ii, jj); - verlet_list.addNeighborLoadBalancing(ii, jj); + // verlet_list.addNeighborNonAtomic(ii, jj); + verlet_list.addNeighborLoadBalancing(ii, jj); } } } @@ -530,7 +530,7 @@ void cabana_short_range( if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); if (jj >= 0) { - //verlet_list.addNeighbor(ii, jj); + // verlet_list.addNeighbor(ii, jj); verlet_list.addNeighborNonAtomic(ii, jj); } } From 98dce63e7e96ccc6663fc3787f7872ef8ba69edc Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 24 Jul 2025 18:33:07 +0200 Subject: [PATCH 65/94] Formatting --- src/core/forces_cabana.hpp | 59 +++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index bc4c894a896..28b8abfdb13 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -38,7 +38,7 @@ struct ForcesKernel { #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ -defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) std::vector unique_particles; #endif [[maybe_unused]] const BondedInteractionsMap bonded_ias; @@ -53,7 +53,7 @@ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) #endif Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ -defined(DPD) or defined(DIPOLES) or defined(NPT) + defined(DPD) or defined(DIPOLES) or defined(NPT) Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; @@ -67,12 +67,12 @@ defined(DPD) or defined(DIPOLES) or defined(NPT) ForcesKernel( // const CellStructure *cell_, #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ -defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) std::vector &unique_particles_, #endif [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, - const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, Kokkos::View local_force_, + const InteractionsNonBonded &nonbonded_ias_, const BoxGeometry &box_geo_, + Kokkos::View local_force_, #ifdef ROTATION Kokkos::View local_torque_, #endif @@ -81,10 +81,9 @@ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) #endif Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ -defined(DPD) or defined(DIPOLES) or defined(NPT) + defined(DPD) or defined(DIPOLES) or defined(NPT) Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const - *elc_kernel_, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, const Thermostat::Thermostat &thermostat_, #endif @@ -92,24 +91,24 @@ defined(DPD) or defined(DIPOLES) or defined(NPT) // int num_threads_), int mpi_rank_, int particle_number_) : // cell(cell_), #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ -defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles(unique_particles_), + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles(unique_particles_), #endif - bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - box_geo(box_geo_), local_force(local_force_), + bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + box_geo(box_geo_), local_force(local_force_), #ifdef ROTATION - local_torque(local_torque_), + local_torque(local_torque_), #endif #ifdef NPT - local_virial(local_virial_), + local_virial(local_virial_), #endif - coulomb_kernel(coulomb_kernel_), + coulomb_kernel(coulomb_kernel_), #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ -defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), - coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), + defined(DPD) or defined(DIPOLES) or defined(NPT) + dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), + coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), #endif - aosoa(aosoa_) { + aosoa(aosoa_) { } KOKKOS_FORCEINLINE_FUNCTION @@ -121,15 +120,15 @@ defined(DPD) or defined(DIPOLES) or defined(NPT) // " " << IA_parameters const &ia_params = - nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); + nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); ParticleForce pf{}; #ifdef NPT Utils::Vector3d virial{}; #endif Utils::Vector3d const d = box_geo.get_mi_vector( - aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), - aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); + aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), + aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); auto const dist = d.norm(); auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); @@ -144,10 +143,10 @@ defined(DPD) or defined(DIPOLES) or defined(NPT) #endif add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, - do_nonbonded_flag, coulomb_kernel); + do_nonbonded_flag, coulomb_kernel); #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ -defined(DPD) or defined(DIPOLES) or defined(NPT) + defined(DPD) or defined(DIPOLES) or defined(NPT) auto const dist2 = dist * dist; #ifndef EXCLUSIONS @@ -155,15 +154,15 @@ defined(DPD) or defined(DIPOLES) or defined(NPT) auto p2 = unique_particles.at(j); #endif // NOT EXCLUSIONS add_non_bonded_pair_force_with_p( - const_cast(*p1), const_cast(*p2), pf, + const_cast(*p1), const_cast(*p2), pf, #ifdef NPT - virial, + virial, #endif // NPT - d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, - box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, - coulomb_u_kernel); + d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, box_geo, + bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, + coulomb_u_kernel); #endif // ETC - // + // local_force(i, thread_id, 0) += pf.f[0]; local_force(i, thread_id, 1) += pf.f[1]; local_force(i, thread_id, 2) += pf.f[2]; From 3344aebb520dc787308a34d024aad9c07a66aabe Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 28 Jul 2025 18:00:11 +0200 Subject: [PATCH 66/94] Small refactoring --- src/core/cabana_data.hpp | 6 +- src/core/custom_verlet_list.hpp | 8 +- src/core/forces.cpp | 27 +++- src/core/forces_cabana.hpp | 85 +++++------ src/core/integrate.cpp | 2 +- src/core/short_range_cabana.hpp | 260 +++----------------------------- 6 files changed, 87 insertions(+), 301 deletions(-) diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp index 7a9bb5e1305..baa5853825f 100644 --- a/src/core/cabana_data.hpp +++ b/src/core/cabana_data.hpp @@ -39,10 +39,10 @@ class CabanaData { int max_id; public: - CabanaData() = default; - CabanaData(ListType verlet_list, std::vector unique_particles) + // CabanaData() = default; + CabanaData(ListType &verlet_list, std::vector &unique_particles) : verlet_list(verlet_list), unique_particles(unique_particles) {} - CabanaData(ListType verlet_list, std::vector unique_particles, + CabanaData(ListType &verlet_list, std::vector &unique_particles, int max_id) : verlet_list(verlet_list), unique_particles(unique_particles), max_id(max_id) {} diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 96627333846..00e497cc960 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -76,13 +76,13 @@ class CustomVerletList nid = tmp; } count = Kokkos::atomic_fetch_add(&counts(pid), 1); - // #ifndef NDEBUG +#ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } - // #endif +#endif neighbors(pid, count) = nid; } @@ -115,13 +115,13 @@ class CustomVerletList nid = tmp; count = counts(pid); } - // #ifndef NDEBUG +#ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( // Kokkos::abort( "Number of count is larger than VerletList size."); } - // #endif +#endif neighbors(pid, count) = nid; counts(pid) += 1; } diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 1624e2a6a94..c30fa3e28f0 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -193,22 +193,33 @@ void System::System::calculate_forces() { }; #ifdef SHARED_MEMORY_PARALLELISM - auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); - auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); - auto elc_kernel_ptr = get_ptr(elc_kernel); - auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); + //auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); + //auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); + //auto elc_kernel_ptr = get_ptr(elc_kernel); + //auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); + ForcesKernel first_neighbor_kernel( + *bonded_ias, *nonbonded_ias, + get_ptr(coulomb_kernel), +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + get_ptr(dipoles_kernel), + get_ptr(elc_kernel), + get_ptr(coulomb_u_kernel), + *thermostat, +#endif + *box_geo); + cabana_short_range( - bond_kernel, *bonded_ias, coulomb_kernel_ptr, dipoles_kernel_ptr, - elc_kernel_ptr, coulomb_u_kernel_ptr, + bond_kernel, first_neighbor_kernel, #ifdef COLLISION_DETECTION collision_detection, #endif *cell_structure, get_interaction_range(), bonded_ias->maximal_cutoff(), - *thermostat, *box_geo, *nonbonded_ias, particles, - cell_structure->ghost_particles(), + particles, cell_structure->ghost_particles(), VerletCriterion<>{*this, cell_structure->get_verlet_skin(), get_interaction_range(), coulomb_cutoff, dipole_cutoff, collision_detection_cutoff}); + #else auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index 28b8abfdb13..628316fb127 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -37,13 +37,21 @@ #include struct ForcesKernel { + [[maybe_unused]] const BondedInteractionsMap &bonded_ias; + const InteractionsNonBonded &nonbonded_ias; + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; + const Thermostat::Thermostat &thermostat; +#endif + const BoxGeometry &box_geo; #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) std::vector unique_particles; #endif - [[maybe_unused]] const BondedInteractionsMap bonded_ias; - const InteractionsNonBonded nonbonded_ias; - const BoxGeometry box_geo; Kokkos::View local_force; #ifdef ROTATION Kokkos::View local_torque; @@ -51,27 +59,36 @@ struct ForcesKernel { #ifdef NPT Kokkos::View local_virial; #endif - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; + AoSoA_pack aosoa; + + ForcesKernel( + [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, + const InteractionsNonBonded &nonbonded_ias_, + Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) or defined(NPT) - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; - const Thermostat::Thermostat &thermostat; + Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, + Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, + Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, + const Thermostat::Thermostat &thermostat_, +#endif + const BoxGeometry &box_geo_) + : + bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + coulomb_kernel(coulomb_kernel_), +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), + coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), #endif - // int num_threads; - // int mpi_rank; - // int particle_number; - const AoSoA_pack aosoa; + box_geo(box_geo_) { + } - ForcesKernel( - // const CellStructure *cell_, + void set_essential_variables( #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) std::vector &unique_particles_, #endif - [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, - const InteractionsNonBonded &nonbonded_ias_, const BoxGeometry &box_geo_, Kokkos::View local_force_, #ifdef ROTATION Kokkos::View local_torque_, @@ -79,45 +96,25 @@ struct ForcesKernel { #ifdef NPT Kokkos::View local_virial_, #endif - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, - const Thermostat::Thermostat &thermostat_, -#endif - const AoSoA_pack &aosoa_) - // int num_threads_), int mpi_rank_, int particle_number_) - : // cell(cell_), + AoSoA_pack &aosoa_) { #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles(unique_particles_), + unique_particles = unique_particles_; #endif - bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - box_geo(box_geo_), local_force(local_force_), + local_force = local_force_; #ifdef ROTATION - local_torque(local_torque_), + local_torque = local_torque_; #endif #ifdef NPT - local_virial(local_virial_), -#endif - coulomb_kernel(coulomb_kernel_), -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), - coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), + local_virial = local_virial_; #endif - aosoa(aosoa_) { + aosoa = aosoa_; } - KOKKOS_FORCEINLINE_FUNCTION - void operator()(int i, int j) const { + __attribute__((always_inline)) KOKKOS_INLINE_FUNCTION void + operator()(int i, int j) const { auto thread_id = omp_get_thread_num(); - // auto thread_id = Kokkos::OpenMP::impl_hardware_thread_id(); - // std::cout << "\nin " << thread_id << "\n"; //" " << i << " " << j << - // " " << IA_parameters const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index 55fc07933d5..e46b6a5aa2c 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -533,7 +533,7 @@ int System::System::integrate(int n_steps, int reuse_forces) { #ifdef SHARED_MEMORY_PARALLELISM else { // cell_structure->set_steepest_descent_flag(true); - cell_structure->set_max_prefactor(7); // 5 + cell_structure->set_max_prefactor(5); // 5 lb, 7 no-lb } #endif auto const calc_md_steps_per_tau = [this](double tau) { diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 3aadb35d390..17aa70b7b1a 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -134,7 +134,7 @@ inline int estimate_max_counts(const double pair_cutoff, using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; -template +template __attribute__((always_inline)) inline void set_verlet_list(CellStructure const &cell_structure, VerletCriterion const &verlet_criterion, Kokkos::View const &id_to_index, @@ -187,8 +187,8 @@ inline void set_verlet_list(CellStructure const &cell_structure, if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); if (jj >= 0) { - // verlet_list.addNeighbor(ii, jj); - verlet_list.addNeighborNonAtomic(ii, jj); + verlet_list.addNeighborAtomic(ii, jj); + // verlet_list.addNeighborNonAtomic(ii, jj); } } } @@ -203,20 +203,15 @@ inline void set_verlet_list(CellStructure const &cell_structure, Kokkos::fence(); } -template +template void cabana_short_range( BondKernel const &bond_kernel, - [[maybe_unused]] BondedInteractionsMap const &bonded_ias, - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel, - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel, + PairKernel &first_neighbor_kernel, #ifdef COLLISION_DETECTION std::shared_ptr collision_detection, #endif CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, - Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, - InteractionsNonBonded const &nonbonded_ias, ParticleRange const &particles, + ParticleRange const &particles, ParticleRange const &ghost_particles, VerletCriterion const &verlet_criterion = {}) { #ifdef CALIPER @@ -261,8 +256,7 @@ void cabana_short_range( cell_structure, number_of_unique_particles, max_id); } else { // If we do not rebuild we can use the saved map - CabanaData saved_data; - saved_data = cell_structure.get_cabana_data(); + CabanaData saved_data = cell_structure.get_cabana_data(); unique_particles = saved_data.get_unique_particles(); number_of_unique_particles = saved_data.get_index(); max_id = saved_data.get_max_id(); @@ -293,152 +287,6 @@ void cabana_short_range( CALI_MARK_END("Cabana - Index map"); #endif - // The kernel of calculate force - struct FirstNeighborKernel { -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - std::vector unique_particles; -#endif - [[maybe_unused]] const BondedInteractionsMap bonded_ias; - const InteractionsNonBonded nonbonded_ias; - const BoxGeometry box_geo; - Kokkos::View local_force; -#ifdef ROTATION - Kokkos::View local_torque; -#endif -#ifdef NPT - Kokkos::View local_virial; -#endif - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; - const Thermostat::Thermostat &thermostat; -#endif - const AoSoA_pack aosoa; - - FirstNeighborKernel( -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - std::vector &unique_particles_, -#endif - [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, - const InteractionsNonBonded &nonbonded_ias_, - const BoxGeometry &box_geo_, Kokkos::View local_force_, -#ifdef ROTATION - Kokkos::View local_torque_, -#endif -#ifdef NPT - Kokkos::View local_virial_, -#endif - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, - Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const - *elc_kernel_, - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, - const Thermostat::Thermostat &thermostat_, -#endif - const AoSoA_pack &aosoa_) - : -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles(unique_particles_), -#endif - bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - box_geo(box_geo_), local_force(local_force_), -#ifdef ROTATION - local_torque(local_torque_), -#endif -#ifdef NPT - local_virial(local_virial_), -#endif - coulomb_kernel(coulomb_kernel_), -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), - coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), -#endif - aosoa(aosoa_) { - } - - KOKKOS_INLINE_FUNCTION - void operator()(int i, int j) const { - - auto thread_id = omp_get_thread_num(); - - IA_parameters const &ia_params = - nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); - - ParticleForce pf{}; -#ifdef NPT - Utils::Vector3d virial{}; -#endif - Utils::Vector3d const d = box_geo.get_mi_vector( - aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), - aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); - auto const dist = d.norm(); - - auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); - -#ifdef EXCLUSIONS - auto p1 = unique_particles.at(i); - auto p2 = unique_particles.at(j); - - bool do_nonbonded_flag = do_nonbonded(*p1, *p2); -#else - bool do_nonbonded_flag = true; -#endif - - add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, - do_nonbonded_flag, coulomb_kernel); - -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - auto const dist2 = dist * dist; - -#ifndef EXCLUSIONS - auto p1 = unique_particles.at(i); - auto p2 = unique_particles.at(j); -#endif // NOT EXCLUSIONS - add_non_bonded_pair_force_with_p( - const_cast(*p1), const_cast(*p2), pf, -#ifdef NPT - virial, -#endif // NPT - d, dist, dist2, q1q2, ia_params, do_nonbonded_flag, thermostat, - box_geo, bonded_ias, coulomb_kernel, dipoles_kernel, elc_kernel, - coulomb_u_kernel); -#endif // ETC - local_force(i, thread_id, 0) += pf.f[0]; - local_force(i, thread_id, 1) += pf.f[1]; - local_force(i, thread_id, 2) += pf.f[2]; -#ifdef ROTATION - local_torque(i, thread_id, 0) += pf.torque[0]; - local_torque(i, thread_id, 1) += pf.torque[1]; - local_torque(i, thread_id, 2) += pf.torque[2]; -#endif - - auto opf = calc_opposing_force(pf, d); - local_force(j, thread_id, 0) += opf.f[0]; - local_force(j, thread_id, 1) += opf.f[1]; - local_force(j, thread_id, 2) += opf.f[2]; -#ifdef ROTATION - local_torque(j, thread_id, 0) += opf.torque[0]; - local_torque(j, thread_id, 1) += opf.torque[1]; - local_torque(j, thread_id, 2) += opf.torque[2]; -#endif -#ifdef NPT - local_virial(thread_id, 0) += virial[0]; - local_virial(thread_id, 1) += virial[1]; - local_virial(thread_id, 2) += virial[2]; -#endif - }; - }; - // Fill the essential variable for MD { #ifdef CALIPER @@ -447,9 +295,8 @@ void cabana_short_range( // =================================================== // Fill particle storage // =================================================== - Kokkos::View id_to_index( - Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); + Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); Kokkos::deep_copy(id_to_index, -1); using policy_type = Kokkos::RangePolicy; @@ -478,72 +325,7 @@ void cabana_short_range( pair_cutoff, number_of_unique_particles, cell_structure); verlet_list = ListType(0, number_of_unique_particles, max_counts); - // set_verlet_list(cell_structure, verlet_criterion, id_to_index, - // verlet_list, max_id); - auto const &cells = - std::as_const(cell_structure).decomposition().local_cells(); - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - - auto kernel_each = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { - auto &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs in this cell */ - for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { - if ((*jt).id() > max_id) - continue; - if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { - int jj = id_to_index((*jt).id()); - if (jj >= 0) { - // verlet_list.addNeighborNonAtomic(ii, jj); - verlet_list.addNeighborLoadBalancing(ii, jj); - } - } - } - } - }; - - auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { - auto &local_particles = cells[i]->particles(); - for (auto it = local_particles.begin(); it != local_particles.end(); - ++it) { - auto const &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs with neighbors */ - for (auto &neighbor : cells[i]->neighbors().red()) { - for (auto const &p2 : neighbor->particles()) { - if (p2.id() > max_id) - continue; - if (verlet_criterion(p1, p2, distance_function(p1, p2))) { - int jj = id_to_index(p2.id()); - if (jj >= 0) { - // verlet_list.addNeighbor(ii, jj); - verlet_list.addNeighborNonAtomic(ii, jj); - } - } - } - } - } - }; - - Kokkos::parallel_for("each", cells.size(), kernel_each); - Kokkos::fence(); - - Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); - Kokkos::fence(); + set_verlet_list(cell_structure, verlet_criterion, id_to_index, verlet_list, max_id); // Save data for next iteration if we just rebuilt CabanaData new_data(verlet_list, unique_particles, max_id); @@ -557,32 +339,28 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); #endif - FirstNeighborKernel first_neighbor_kernel( - // ForcesKernel first_neighbor_kernel( + first_neighbor_kernel.set_essential_variables( #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles, + unique_particles, #endif - bonded_ias, nonbonded_ias, box_geo, local_force, + local_force, #ifdef ROTATION - local_torque, + local_torque, #endif #ifdef NPT - local_virial, -#endif - coulomb_kernel, -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel, elc_kernel, coulomb_u_kernel, thermostat, + local_virial, #endif - aosoa); + aosoa); - const auto &kernel_force = first_neighbor_kernel; + //const auto kernel_force = first_neighbor_kernel; + const auto kernel_force = std::move(first_neighbor_kernel); // verlet_list.get_variance_max_counts(); Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, kernel_force, verlet_list, Cabana::FirstNeighborsTag(), - Cabana::TeamOpTag()); + //Cabana::TeamOpTag()); + Cabana::SerialOpTag()); Kokkos::fence(); #ifdef CALIPER CALI_MARK_END("Cabana - calc Force"); From cdafecfe31c92bf89ddea351c52d9953a3bfafd5 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Mon, 28 Jul 2025 18:01:37 +0200 Subject: [PATCH 67/94] Formatting --- src/core/forces.cpp | 15 +++++-------- src/core/forces_cabana.hpp | 5 ++--- src/core/short_range_cabana.hpp | 39 +++++++++++++++++---------------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/core/forces.cpp b/src/core/forces.cpp index c30fa3e28f0..8bf1c3445e2 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -193,18 +193,15 @@ void System::System::calculate_forces() { }; #ifdef SHARED_MEMORY_PARALLELISM - //auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); - //auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); - //auto elc_kernel_ptr = get_ptr(elc_kernel); - //auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); + // auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); + // auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); + // auto elc_kernel_ptr = get_ptr(elc_kernel); + // auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); ForcesKernel first_neighbor_kernel( - *bonded_ias, *nonbonded_ias, - get_ptr(coulomb_kernel), + *bonded_ias, *nonbonded_ias, get_ptr(coulomb_kernel), #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) or defined(NPT) - get_ptr(dipoles_kernel), - get_ptr(elc_kernel), - get_ptr(coulomb_u_kernel), + get_ptr(dipoles_kernel), get_ptr(elc_kernel), get_ptr(coulomb_u_kernel), *thermostat, #endif *box_geo); diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index 628316fb127..1f8841d554d 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -73,15 +73,14 @@ struct ForcesKernel { const Thermostat::Thermostat &thermostat_, #endif const BoxGeometry &box_geo_) - : - bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + : bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), coulomb_kernel(coulomb_kernel_), #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ defined(DPD) or defined(DIPOLES) or defined(NPT) dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), #endif - box_geo(box_geo_) { + box_geo(box_geo_) { } void set_essential_variables( diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 17aa70b7b1a..e8c8b8cd595 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -134,11 +134,12 @@ inline int estimate_max_counts(const double pair_cutoff, using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; -template __attribute__((always_inline)) -inline void set_verlet_list(CellStructure const &cell_structure, - VerletCriterion const &verlet_criterion, - Kokkos::View const &id_to_index, - ListType &verlet_list, const int max_id) { +template +__attribute__((always_inline)) inline void +set_verlet_list(CellStructure const &cell_structure, + VerletCriterion const &verlet_criterion, + Kokkos::View const &id_to_index, ListType &verlet_list, + const int max_id) { auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); auto const distance_function = detail::MinimalImageDistance{ @@ -203,16 +204,15 @@ inline void set_verlet_list(CellStructure const &cell_structure, Kokkos::fence(); } -template +template void cabana_short_range( - BondKernel const &bond_kernel, - PairKernel &first_neighbor_kernel, + BondKernel const &bond_kernel, PairKernel &first_neighbor_kernel, #ifdef COLLISION_DETECTION std::shared_ptr collision_detection, #endif CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, - ParticleRange const &particles, - ParticleRange const &ghost_particles, + ParticleRange const &particles, ParticleRange const &ghost_particles, VerletCriterion const &verlet_criterion = {}) { #ifdef CALIPER CALI_CXX_MARK_FUNCTION; @@ -296,7 +296,7 @@ void cabana_short_range( // Fill particle storage // =================================================== Kokkos::View id_to_index( - Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); + Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); Kokkos::deep_copy(id_to_index, -1); using policy_type = Kokkos::RangePolicy; @@ -325,7 +325,8 @@ void cabana_short_range( pair_cutoff, number_of_unique_particles, cell_structure); verlet_list = ListType(0, number_of_unique_particles, max_counts); - set_verlet_list(cell_structure, verlet_criterion, id_to_index, verlet_list, max_id); + set_verlet_list(cell_structure, verlet_criterion, id_to_index, + verlet_list, max_id); // Save data for next iteration if we just rebuilt CabanaData new_data(verlet_list, unique_particles, max_id); @@ -342,24 +343,24 @@ void cabana_short_range( first_neighbor_kernel.set_essential_variables( #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles, + unique_particles, #endif - local_force, + local_force, #ifdef ROTATION - local_torque, + local_torque, #endif #ifdef NPT - local_virial, + local_virial, #endif - aosoa); + aosoa); - //const auto kernel_force = first_neighbor_kernel; + // const auto kernel_force = first_neighbor_kernel; const auto kernel_force = std::move(first_neighbor_kernel); // verlet_list.get_variance_max_counts(); Kokkos::RangePolicy policy(0, particle_storage.size()); Cabana::neighbor_parallel_for(policy, kernel_force, verlet_list, Cabana::FirstNeighborsTag(), - //Cabana::TeamOpTag()); + // Cabana::TeamOpTag()); Cabana::SerialOpTag()); Kokkos::fence(); #ifdef CALIPER From 9e36c08c98b8ae30c537a4d7731b7e7f6cf74a8d Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 1 Aug 2025 17:44:16 +0200 Subject: [PATCH 68/94] Refactoring --- src/core/aosoa_pack.hpp | 10 +- src/core/cell_system/CellStructure.cpp | 87 ++++++- src/core/cell_system/CellStructure.hpp | 139 ++++++++-- src/core/custom_verlet_list.hpp | 37 +-- src/core/forces.cpp | 43 ++- src/core/forces_cabana.hpp | 58 ++--- src/core/integrate.cpp | 4 +- src/core/short_range_cabana.hpp | 346 +++++++++---------------- 8 files changed, 397 insertions(+), 327 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index c7d85e84104..79f87fab229 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -23,7 +23,7 @@ #include -const int vector_length = 1; +//const int vector_length = 1; using data_types = Cabana::MemberTypes; //, bool>; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; @@ -31,20 +31,14 @@ using AoSoA_type = Cabana::AoSoA; struct AoSoA_pack { AoSoA_type::member_slice_type<0> position; - // AoSoA_type::member_slice_type<1> force; - // AoSoA_type::member_slice_type<2> torque; AoSoA_type::member_slice_type<1> charge; AoSoA_type::member_slice_type<2> id; AoSoA_type::member_slice_type<3> type; - // AoSoA_type::member_slice_type<4> ghost; AoSoA_pack() = default; AoSoA_pack(AoSoA_type &aosoa) - : // position(Cabana::slice<0>(aosoa)), force(Cabana::slice<1>(aosoa)), - // torque(Cabana::slice<2>(aosoa)), charge(Cabana::slice<3>(aosoa)), - position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), + : position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)) {} - // ghost(Cabana::slice<4>(aosoa)) {} }; #endif diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 96026ca6cd6..ce6a7befe44 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -52,25 +52,49 @@ #include #ifdef SHARED_MEMORY_PARALLELISM +#include "aosoa_pack.hpp" #include "cabana_data.hpp" #include "custom_verlet_list.hpp" #include +#include #include #endif #ifdef SHARED_MEMORY_PARALLELISM -using memory_space = Kokkos::SharedSpace; -using execution_space = Kokkos::DefaultExecutionSpace; +//using memory_space = Kokkos::HostSpace; +//using execution_space = Kokkos::DefaultExecutionSpace; -using ListAlgorithm = Cabana::HalfNeighborTag; -using ListType = Cabana::CustomVerletList; +//using ListAlgorithm = Cabana::HalfNeighborTag; +//using ListType = Cabana::CustomVerletList; CellStructure::~CellStructure() { if (m_cabana_data) { m_cabana_data.reset(); } + if (m_local_force) { + m_local_force.reset(); + } +#ifdef ROTATION + if (m_local_torque) { + m_local_torque.reset(); + } +#endif +#ifdef NPT + if (m_local_virial) { + m_local_virial.reset(); + } +#endif + if (m_aosoa) { + m_aosoa.reset(); + } + if (m_particle_storage) { + m_particle_storage.reset(); + } + if (m_cabana_verlet_list) { + m_cabana_verlet_list.reset(); + } } void CellStructure::set_cabana_data(std::unique_ptr data) { @@ -85,8 +109,61 @@ void CellStructure::reset_cabana_data() { if (m_cabana_data) { m_cabana_data.reset(); } + if (m_local_force) { + m_local_force.reset(); + } +#ifdef ROTATION + if (m_local_torque) { + m_local_torque.reset(); + } +#endif +#ifdef NPT + if (m_local_virial) { + m_local_virial.reset(); + } +#endif + if (m_aosoa) { + m_aosoa.reset(); + } + if (m_particle_storage) { + m_particle_storage.reset(); + } + if (m_cabana_verlet_list) { + m_cabana_verlet_list.reset(); + } } +void CellStructure::rebuild_local_properties(const std::size_t num_part, const std::size_t num_threads, const double pair_cutoff) { + m_local_force = std::make_unique> + ("local_force", num_part, num_threads); +#ifdef ROTATION + m_local_torque = std::make_unique> + ("local_torque", num_part, num_threads); +#endif +#ifdef NPT + m_local_virial = std::make_unique> + ("local_virial", num_threads); +#endif + m_particle_storage = + std::make_unique> + ("particles", num_part); + (*m_particle_storage).resize(num_part); + // particle properties are defined in aosoa_pack.hpp + m_aosoa = std::make_unique(*m_particle_storage); + + int max_counts = estimate_max_counts(pair_cutoff, num_part); + m_cabana_verlet_list = std::make_unique(0, num_part, max_counts); +} + +void CellStructure::reset_local_properties() { + Kokkos::deep_copy(get_local_force(), 0); +#ifdef ROTATION + Kokkos::deep_copy(get_local_torque(), 0); +#endif +#ifdef NPT + Kokkos::deep_copy(get_local_virial(), 0); +#endif +} #endif CellStructure::CellStructure(BoxGeometry const &box) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index beef57b5ade..4abbe36c783 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -53,11 +53,39 @@ #include #include #include +#include #include +#ifdef CALIPER +#include +#endif + // forward declaration to not have to import cabana #ifdef SHARED_MEMORY_PARALLELISM +namespace Kokkos { + template + class View; + class HostSpace; + class LayoutRight; + template + class MemoryTraits; +} +namespace Cabana { + class HalfNeighborTag; + class VerletLayout2D; + class TeamVectorOpTag; + template + class CustomVerletList; + template + struct MemberTypes; + template + class AoSoA; +} class CabanaData; +struct AoSoA_pack; +// To construct AoSoA, vector_length is defined HERE. +const int vector_length = 1; #endif template @@ -164,6 +192,28 @@ struct CellStructure : public System::Leaf { double m_verlet_skin = 0.; bool m_verlet_skin_set = false; double m_verlet_reuse = 0.; +#ifdef SHARED_MEMORY_PARALLELISM + std::unique_ptr> m_local_force; +#ifdef ROTATION + std::unique_ptr> m_local_torque; +#endif +#ifdef NPT + std::unique_ptr> m_local_virial; +#endif + using data_types = Cabana::MemberTypes; //, bool>; + using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; + using AoSoAType = Cabana::AoSoA>; + std::unique_ptr m_particle_storage; + /** particle properties for Cabana defined in aosoa_pack.hpp */ + std::unique_ptr m_aosoa; + /** The local id-to-index for aosoa data */ + std::vector m_unique_particles; + + using ListAlgorithm = Cabana::HalfNeighborTag; + using ListType = Cabana::CustomVerletList; + std::unique_ptr m_cabana_verlet_list; +#endif public: CellStructure(BoxGeometry const &box); @@ -664,6 +714,29 @@ struct CellStructure : public System::Leaf { // bool steepest_descent_flag = true; std::size_t max_prefactor = 8; std::size_t max_counts = -1; + int m_max_id = 0; + + inline int estimate_max_counts(const double pair_cutoff, + const int number_of_unique_particles) { + //std::cout << "estimate_max_counts:" << pair_cutoff << " " +// << max_prefactor << std::endl; + int max_counts; + if (not std::isinf(pair_cutoff)) { + max_counts = + static_cast(std::ceil(max_prefactor * + pair_cutoff * pair_cutoff * pair_cutoff)); + int threshold_num = 16; //8; +#ifdef COLLISION_DETECTION + threshold_num = 64; +#endif + if (max_counts < threshold_num) { + max_counts = std::min(threshold_num, number_of_unique_particles); + } + } else { + max_counts = number_of_unique_particles; + } + return max_counts; + } public: void set_cabana_data(std::unique_ptr data); @@ -676,6 +749,9 @@ struct CellStructure : public System::Leaf { bool get_rebuild_cabana_verlet_list() const { return m_rebuild_cabana_verlet_list; } + void mark_rebuild_cabana_verlet_list_as_UpToDate() { + m_rebuild_cabana_verlet_list = false; + } // void set_steepest_descent_flag(bool flag) { steepest_descent_flag = flag; } // bool get_steepest_descent_flag() { return steepest_descent_flag; } @@ -686,31 +762,52 @@ struct CellStructure : public System::Leaf { void set_max_counts(std::size_t value) { max_counts = value; } std::size_t get_max_counts() { return max_counts; } - template void cabana_link_cell(Kernel kernel) { - auto const local_cells_span = decomposition().local_cells(); - auto const first = boost::make_indirect_iterator(local_cells_span.begin()); - auto const last = boost::make_indirect_iterator(local_cells_span.end()); + int get_max_id() { return m_max_id; } - Algorithm::link_cell( - first, last, [&kernel](Particle &p1, Particle &p2) { kernel(p1, p2); }); - } + void rebuild_local_properties(std::size_t num_part, std::size_t num_threads, double pair_cutoff); + void reset_local_properties(); - template - void cabana_verlet_list_loop(Kernel kernel, - const VerletCriterion &verlet_criterion) { - if (m_rebuild_cabana_verlet_list) { - // if (m_rebuild_verlet_list) { - m_verlet_list.clear(); + Kokkos::View& get_local_force() { return *m_local_force; } +#ifdef ROTATION + Kokkos::View& get_local_torque() { return *m_local_torque; } +#endif +#ifdef NPT + Kokkos::View& get_local_virial() { return *m_local_virial; } +#endif + AoSoA_pack& get_aosoa_data() { return *m_aosoa; }; + ListType& get_cabana_verlet_list() { return *m_cabana_verlet_list; }; + std::vector& get_unique_particles() { return m_unique_particles; } + + inline void set_index_map(ParticleRange const &particles, + ParticleRange const &ghost_particles, + int &index) { + m_unique_particles.clear(); + m_max_id = 0; + std::unordered_set registered_index{}; + for (auto &p : particles) { + if (p.id() > m_max_id) + m_max_id = p.id(); + m_unique_particles.emplace_back(&p); + index++; + } - link_cell([&](Particle &p1, Particle &p2, Distance const &d) { - if (verlet_criterion(p1, p2, d)) { - m_verlet_list.emplace_back(&p1, &p2); - kernel(p1, p2); - } - }); - m_rebuild_verlet_list = false; - m_rebuild_cabana_verlet_list = false; + for (auto &p : ghost_particles) { + if (not get_local_particle(p.id())) { + continue; + } + if (not get_local_particle(p.id())->is_ghost()) { + continue; + } + if (registered_index.contains(p.id())) { + continue; + } + if (p.id() > m_max_id) + m_max_id = p.id(); + registered_index.insert(p.id()); + m_unique_particles.emplace_back(&p); + index++; } + registered_index.clear(); } #endif diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 00e497cc960..856fbc40a52 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -21,6 +21,7 @@ #ifdef SHARED_MEMORY_PARALLELISM #include +#include namespace Cabana { // ONLY FOR 2D LAYOUT, OTHERWISE NEIGHBOR LIST INTERFACE IMPLEMENTATION WILL @@ -36,29 +37,22 @@ class CustomVerletList CustomVerletList() : Base() {} // Custom constructor - // template - // CustomVerletList(PositionSlice x, const std::size_t begin, CustomVerletList(const std::size_t begin, const std::size_t end, const std::size_t max_neigh) { - // const std::size_t thread_number) { - // initializeData(x.size(), max_neigh);//, thread_number); - initializeData(end - begin, max_neigh); //, thread_number); + initializeData(end - begin, max_neigh); } virtual ~CustomVerletList() {}; public: Kokkos::View counts; - Kokkos::View neighbors; - // Kokkos::View neighbors; + Kokkos::View neighbors; // Method to initialize _data without filling neighbors KOKKOS_INLINE_FUNCTION void initializeData(const std::size_t num_particles, const std::size_t max_neigh) { - // const std::size_t thread_number) { counts = Kokkos::View("num_neighbors", num_particles); - // neighbors = Kokkos::View( - neighbors = Kokkos::View( + neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, max_neigh); } @@ -70,7 +64,6 @@ class CustomVerletList std::size_t count_n = counts(nid); if (count > count_n) { - // if (pid > nid) { int tmp = pid; pid = nid; nid = tmp; @@ -79,7 +72,6 @@ class CustomVerletList #ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( - // Kokkos::abort( "Number of count is larger than VerletList size."); } #endif @@ -91,13 +83,12 @@ class CustomVerletList void addNeighborNonAtomic(int pid, int nid) { std::size_t count = counts(pid); - // #ifndef NDEBUG +#ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( - // Kokkos::abort( "Number of count is larger than VerletList size."); } - // #endif +#endif neighbors(pid, count) = nid; counts(pid) += 1; } @@ -109,7 +100,6 @@ class CustomVerletList std::size_t count_n = counts(nid); if (count > count_n) { - // if (pid > nid) { int tmp = pid; pid = nid; nid = tmp; @@ -118,7 +108,6 @@ class CustomVerletList #ifndef NDEBUG if (count >= neighbors.extent(1)) { throw std::runtime_error( - // Kokkos::abort( "Number of count is larger than VerletList size."); } #endif @@ -126,6 +115,20 @@ class CustomVerletList counts(pid) += 1; } + // Sorting a neighbor + KOKKOS_INLINE_FUNCTION + void sortNeighbors() { + Kokkos::parallel_for( + "custom_velet_list::sort_neighbors", + Kokkos::RangePolicy(0, counts.size()), + [&](const int i) { + const int count = counts(i); + int* ptr = &neighbors(i, 0); + std::sort(ptr, ptr + count); + }); + Kokkos::fence(); + } + // Find max counts KOKKOS_INLINE_FUNCTION std::size_t get_variance_max_counts() { diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 8bf1c3445e2..d986d79cd36 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -193,10 +193,25 @@ void System::System::calculate_forces() { }; #ifdef SHARED_MEMORY_PARALLELISM - // auto coulomb_kernel_ptr = get_ptr(coulomb_kernel); - // auto dipoles_kernel_ptr = get_ptr(dipoles_kernel); - // auto elc_kernel_ptr = get_ptr(elc_kernel); - // auto coulomb_u_kernel_ptr = get_ptr(coulomb_u_kernel); + auto const &verlet_criterion = + VerletCriterion<>{*this, cell_structure->get_verlet_skin(), + get_interaction_range(), coulomb_cutoff, dipole_cutoff, + collision_detection_cutoff}; + update_cabana_state(*cell_structure, particles, cell_structure->ghost_particles(), + verlet_criterion, get_interaction_range()); +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + auto unique_particles = cell_structure->get_unique_particles(); +#endif + auto local_force = cell_structure->get_local_force(); +#ifdef ROTATION + auto local_torque = cell_structure->get_local_torque(); +#endif +#ifdef NPT + auto local_virial = cell_structure->get_local_virial(); +#endif + auto const &aosoa = cell_structure->get_aosoa_data(); + ForcesKernel first_neighbor_kernel( *bonded_ias, *nonbonded_ias, get_ptr(coulomb_kernel), #if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ @@ -204,7 +219,19 @@ void System::System::calculate_forces() { get_ptr(dipoles_kernel), get_ptr(elc_kernel), get_ptr(coulomb_u_kernel), *thermostat, #endif - *box_geo); + *box_geo, +#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + unique_particles, +#endif + local_force, +#ifdef ROTATION + local_torque, +#endif +#ifdef NPT + local_virial, +#endif + aosoa); cabana_short_range( bond_kernel, first_neighbor_kernel, @@ -212,11 +239,7 @@ void System::System::calculate_forces() { collision_detection, #endif *cell_structure, get_interaction_range(), bonded_ias->maximal_cutoff(), - particles, cell_structure->ghost_particles(), - VerletCriterion<>{*this, cell_structure->get_verlet_skin(), - get_interaction_range(), coulomb_cutoff, dipole_cutoff, - collision_detection_cutoff}); - + particles, cell_structure->ghost_particles(), verlet_criterion); #else auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index 1f8841d554d..a55b6b95ce8 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -26,15 +26,8 @@ #ifdef SHARED_MEMORY_PARALLELISM #include "aosoa_pack.hpp" -#include "cabana_data.hpp" -#include "custom_verlet_list.hpp" +#include "forces_inline.hpp" #include -#include -#include -#include -#include -#include -#include struct ForcesKernel { [[maybe_unused]] const BondedInteractionsMap &bonded_ias; @@ -50,16 +43,16 @@ struct ForcesKernel { const BoxGeometry &box_geo; #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - std::vector unique_particles; + std::vector &unique_particles; #endif - Kokkos::View local_force; + Kokkos::View local_force; #ifdef ROTATION - Kokkos::View local_torque; + Kokkos::View local_torque; #endif #ifdef NPT - Kokkos::View local_virial; + Kokkos::View local_virial; #endif - AoSoA_pack aosoa; + const AoSoA_pack &aosoa; ForcesKernel( [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, @@ -72,42 +65,39 @@ struct ForcesKernel { Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, const Thermostat::Thermostat &thermostat_, #endif - const BoxGeometry &box_geo_) - : bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), - coulomb_kernel(coulomb_kernel_), -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) - dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), - coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), -#endif - box_geo(box_geo_) { - } - - void set_essential_variables( + const BoxGeometry &box_geo_, #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) std::vector &unique_particles_, #endif - Kokkos::View local_force_, + Kokkos::View local_force_, #ifdef ROTATION - Kokkos::View local_torque_, + Kokkos::View local_torque_, #endif #ifdef NPT - Kokkos::View local_virial_, + Kokkos::View local_virial_, +#endif + const AoSoA_pack &aosoa_) + : bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), + coulomb_kernel(coulomb_kernel_), +#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ + defined(DPD) or defined(DIPOLES) or defined(NPT) + dipoles_kernel(dipoles_kernel_), elc_kernel(elc_kernel_), + coulomb_u_kernel(coulomb_u_kernel_), thermostat(thermostat_), #endif - AoSoA_pack &aosoa_) { + box_geo(box_geo_), #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles = unique_particles_; + unique_particles(unique_particles_), #endif - local_force = local_force_; + local_force(local_force_), #ifdef ROTATION - local_torque = local_torque_; + local_torque(local_torque_), #endif #ifdef NPT - local_virial = local_virial_; + local_virial(local_virial_), #endif - aosoa = aosoa_; + aosoa(aosoa_) { } __attribute__((always_inline)) KOKKOS_INLINE_FUNCTION void diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index e46b6a5aa2c..f52884572e7 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -527,13 +527,13 @@ int System::System::integrate(int n_steps, int reuse_forces) { ek_active = ek.is_ready_for_propagation(); #ifdef SHARED_MEMORY_PARALLELISM // cell_structure->set_steepest_descent_flag(false); - cell_structure->set_max_prefactor(8); + cell_structure->set_max_prefactor(5); #endif } #ifdef SHARED_MEMORY_PARALLELISM else { // cell_structure->set_steepest_descent_flag(true); - cell_structure->set_max_prefactor(5); // 5 lb, 7 no-lb + cell_structure->set_max_prefactor(8); } #endif auto const calc_md_steps_per_tau = [this](double tau) { diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index e8c8b8cd595..1dc544a74b7 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -22,7 +22,6 @@ #include "config/config.hpp" #include "cell_system/CellStructure.hpp" -// #include "lees_edwards/lees_edwards.hpp" #ifdef CALIPER #include @@ -31,23 +30,14 @@ #ifdef SHARED_MEMORY_PARALLELISM #include "aosoa_pack.hpp" -#include "cabana_data.hpp" +//#include "cabana_data.hpp" #include "custom_verlet_list.hpp" #include "forces_cabana.hpp" #include #include #include -#include -inline double wrap(double x, double L) { - auto result = x - std::floor(x / L) * L; - // if (result >= L) - // result -= std::nextafter(L, 0.); - return result; -} - -inline void write_particle(Particle const &p, int const &id, - AoSoA_pack &aosoa) { +inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); @@ -57,96 +47,23 @@ inline void write_particle(Particle const &p, int const &id, } } -inline void write_particle_permute(Particle const &p, int const &id, - AoSoA_pack &aosoa, - Utils::Vector3d const &box_l) { - aosoa.id(id) = p.id(); - aosoa.charge(id) = p.q(); - aosoa.type(id) = p.type(); - auto const pos = p.pos(); - double wpos[3] = {}; - for (int d = 0; d < 3; ++d) { - // aosoa.position(id, d) = - // pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; - wpos[d] = pos[d] - std::floor(pos[d] / box_l[d]) * box_l[d]; - } - for (int d = 0; d < 3; ++d) { - aosoa.position(id, d) = wpos[d]; - } - // assert(aosoa.position(id, 0) >= 0. and aosoa.position(id, 0) < box_l[0]); - // assert(aosoa.position(id, 1) >= 0. and aosoa.position(id, 1) < box_l[1]); - // assert(aosoa.position(id, 2) >= 0. and aosoa.position(id, 2) < box_l[2]); -} - -inline void set_index_map(std::vector &unique_particles, - ParticleRange const &particles, - ParticleRange const &ghost_particles, - CellStructure const &cell_structure, int &index, - int &max_id) { - std::unordered_set registered_index{}; - for (auto &p : particles) { - if (p.id() > max_id) - max_id = p.id(); - unique_particles.emplace_back(&p); - index++; - } - - for (auto &p : ghost_particles) { - if (not cell_structure.get_local_particle(p.id())) { - continue; - } - if (not cell_structure.get_local_particle(p.id())->is_ghost()) { - continue; - } - if (registered_index.contains(p.id())) { - continue; - } - if (p.id() > max_id) - max_id = p.id(); - registered_index.insert(p.id()); - unique_particles.emplace_back(&p); - index++; - } - registered_index.clear(); -} - -inline int estimate_max_counts(const double pair_cutoff, - const int number_of_unique_particles, - CellStructure &cell_structure) { - int max_counts; - if (not std::isinf(pair_cutoff)) { - max_counts = - static_cast(std::ceil(cell_structure.get_max_prefactor() * - pair_cutoff * pair_cutoff * pair_cutoff)); - int threshold_num = 8; -#ifdef COLLISION_DETECTION - threshold_num = 64; -#endif - if (max_counts < threshold_num) { - max_counts = std::min(threshold_num, number_of_unique_particles); - } - } else { - max_counts = number_of_unique_particles; - } - return max_counts; -} - using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; -template +template __attribute__((always_inline)) inline void -set_verlet_list(CellStructure const &cell_structure, - VerletCriterion const &verlet_criterion, - Kokkos::View const &id_to_index, ListType &verlet_list, - const int max_id) { +construct_verlet_list(CellStructure &cell_structure, + VerletCriterion const &verlet_criterion, + Kokkos::View const &id_to_index, + const int max_id) { auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; + auto verlet_list = cell_structure.get_cabana_verlet_list(); - auto kernel_each = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { + auto intra_kernel = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](const int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto &p1 = *it; @@ -162,7 +79,6 @@ set_verlet_list(CellStructure const &cell_structure, if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { int jj = id_to_index((*jt).id()); if (jj >= 0) { - // verlet_list.addNeighborNonAtomic(ii, jj); verlet_list.addNeighborLoadBalancing(ii, jj); } } @@ -170,8 +86,8 @@ set_verlet_list(CellStructure const &cell_structure, } }; - auto kernel_neighbor = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](int i) { + auto inter_kernel = [&cells, &distance_function, &verlet_criterion, + &id_to_index, &verlet_list, max_id](const int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto const &p1 = *it; @@ -188,8 +104,8 @@ set_verlet_list(CellStructure const &cell_structure, if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); if (jj >= 0) { - verlet_list.addNeighborAtomic(ii, jj); - // verlet_list.addNeighborNonAtomic(ii, jj); + // verlet_list.addNeighborAtomic(ii, jj); + verlet_list.addNeighborNonAtomic(ii, jj); } } } @@ -197,182 +113,150 @@ set_verlet_list(CellStructure const &cell_structure, } }; - Kokkos::parallel_for("each", cells.size(), kernel_each); + Kokkos::parallel_for("inter", cells.size(), intra_kernel); Kokkos::fence(); - Kokkos::parallel_for("neighbor", cells.size(), kernel_neighbor); + Kokkos::parallel_for("intra", cells.size(), inter_kernel); Kokkos::fence(); + + // verlet_list.sortNeighbors(); } -template -void cabana_short_range( - BondKernel const &bond_kernel, PairKernel &first_neighbor_kernel, -#ifdef COLLISION_DETECTION - std::shared_ptr collision_detection, -#endif - CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, - ParticleRange const &particles, ParticleRange const &ghost_particles, - VerletCriterion const &verlet_criterion = {}) { +template +__attribute__((always_inline)) inline void +update_cabana_state(CellStructure &cell_structure, + ParticleRange const &particles, + ParticleRange const &ghost_particles, + VerletCriterion const &verlet_criterion, + double const pair_cutoff) { #ifdef CALIPER - CALI_CXX_MARK_FUNCTION; + CALI_MARK_BEGIN("Cabana - Index map"); #endif + // Number of threads + int num_threads = execution_space().concurrency(); -#ifdef CALIPER - CALI_MARK_BEGIN("Espresso - Bond Kernel"); -#endif - assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); + int number_of_unique_particles = 0; - if (bond_cutoff >= 0.) { - cell_structure.bond_loop(bond_kernel); + bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); + // std::cout << "rebuild:" << rebuild << std::endl; + + if (rebuild) { + // If we have to rebuild, we need to count the particles + cell_structure.set_index_map(particles, ghost_particles, number_of_unique_particles); + // Create essential variable for MD + cell_structure.rebuild_local_properties(number_of_unique_particles, num_threads, pair_cutoff); + } else { + // If we do not rebuild we can use the saved map + number_of_unique_particles = cell_structure.get_unique_particles().size(); + cell_structure.reset_local_properties(); } + auto const unique_particles = cell_structure.get_unique_particles(); + auto aosoa = cell_structure.get_aosoa_data(); + int max_id = cell_structure.get_max_id(); + #ifdef CALIPER - CALI_MARK_END("Espresso - Bond Kernel"); + CALI_MARK_END("Cabana - Index map"); +#endif + // Fill the essential variable for MD + { +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Allocation"); #endif - - // Cabana short range loop - if (pair_cutoff > 0.) { // =================================================== - // Count unique particles and create Index map + // Fill particle storage // =================================================== + Kokkos::View id_to_index( + Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); + Kokkos::deep_copy(id_to_index, -1); + + using policy_type = Kokkos::RangePolicy; + Kokkos::parallel_for( + "AoSoA write", policy_type(0, number_of_unique_particles), + [&unique_particles, &aosoa, &id_to_index](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, aosoa); + id_to_index(unique_particles.at(p_id)->id()) = p_id; + }); + Kokkos::fence(); #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Index map"); + CALI_MARK_END("Cabana - Allocation"); #endif - // Number of threads - int num_threads = execution_space().concurrency(); - - std::vector unique_particles; - int number_of_unique_particles = 0; - int max_id = 0; - - bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or - (not cell_structure.use_verlet_list); - - ListType verlet_list; - - if (rebuild) { - // If we have to rebuild, we need to count the particles - set_index_map(unique_particles, particles, ghost_particles, - cell_structure, number_of_unique_particles, max_id); - } else { - // If we do not rebuild we can use the saved map - CabanaData saved_data = cell_structure.get_cabana_data(); - unique_particles = saved_data.get_unique_particles(); - number_of_unique_particles = saved_data.get_index(); - max_id = saved_data.get_max_id(); - verlet_list = saved_data.get_verlet_list(); - } // =================================================== - // Create essential variable for MD + // Get Verlet Pairs and Fill Verlet list // =================================================== - Kokkos::View local_force( - "local_force", number_of_unique_particles, num_threads); - -#ifdef ROTATION - Kokkos::View local_torque( - "local_torque", number_of_unique_particles, num_threads); -#endif -#ifdef NPT - Kokkos::View local_virial("local_virial", - num_threads); -#endif - Cabana::AoSoA particle_storage( - "particles", number_of_unique_particles); - particle_storage.resize(number_of_unique_particles); - // particle properties are defined in aosoa_pack.hpp - auto aosoa = AoSoA_pack(particle_storage); + // Rebuild verlet list if needed + if (rebuild) { #ifdef CALIPER - CALI_MARK_END("Cabana - Index map"); + CALI_MARK_BEGIN("Cabana - Verlet List"); #endif - - // Fill the essential variable for MD - { + construct_verlet_list(cell_structure, verlet_criterion, id_to_index, max_id); + cell_structure.mark_rebuild_cabana_verlet_list_as_UpToDate(); #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Allocation"); + CALI_MARK_END("Cabana - Verlet List"); #endif - // =================================================== - // Fill particle storage - // =================================================== - Kokkos::View id_to_index( - Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); - Kokkos::deep_copy(id_to_index, -1); - - using policy_type = Kokkos::RangePolicy; - Kokkos::parallel_for( - "AoSoA write", policy_type(0, particle_storage.size()), - [&unique_particles, &aosoa, &id_to_index](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, aosoa); - id_to_index(unique_particles.at(p_id)->id()) = p_id; - }); - Kokkos::fence(); + } + } +} +template +void cabana_short_range( + BondKernel const &bond_kernel, PairKernel const &forces_kernel, +#ifdef COLLISION_DETECTION + std::shared_ptr collision_detection, +#endif + CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, + ParticleRange const &particles, ParticleRange const &ghost_particles, + VerletCriterion const &verlet_criterion = {}) { #ifdef CALIPER - CALI_MARK_END("Cabana - Allocation"); + CALI_CXX_MARK_FUNCTION; #endif - // =================================================== - // Get Verlet Pairs and Fill Verlet list - // =================================================== + int num_threads = execution_space().concurrency(); - // Rebuild verlet list if needed - if (rebuild) { #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List"); + CALI_MARK_BEGIN("Espresso - Bond Kernel"); #endif - int max_counts = estimate_max_counts( - pair_cutoff, number_of_unique_particles, cell_structure); - verlet_list = ListType(0, number_of_unique_particles, max_counts); - - set_verlet_list(cell_structure, verlet_criterion, id_to_index, - verlet_list, max_id); + assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); - // Save data for next iteration if we just rebuilt - CabanaData new_data(verlet_list, unique_particles, max_id); - cell_structure.set_cabana_data(std::make_unique(new_data)); + if (bond_cutoff >= 0.) { + cell_structure.bond_loop(bond_kernel); + } #ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List"); + CALI_MARK_END("Espresso - Bond Kernel"); #endif - } - } - { + + // Cabana short range loop + if (pair_cutoff > 0.) { #ifdef CALIPER - CALI_MARK_BEGIN("Cabana - calc Force"); -#endif - first_neighbor_kernel.set_essential_variables( -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles, + CALI_MARK_BEGIN("Cabana - calc Force"); #endif - local_force, + auto unique_particles = cell_structure.get_unique_particles(); + auto local_force = cell_structure.get_local_force(); #ifdef ROTATION - local_torque, + auto local_torque = cell_structure.get_local_torque(); #endif #ifdef NPT - local_virial, + auto local_virial = cell_structure.get_local_virial(); #endif - aosoa); - - // const auto kernel_force = first_neighbor_kernel; - const auto kernel_force = std::move(first_neighbor_kernel); - // verlet_list.get_variance_max_counts(); - Kokkos::RangePolicy policy(0, particle_storage.size()); - Cabana::neighbor_parallel_for(policy, kernel_force, verlet_list, - Cabana::FirstNeighborsTag(), - // Cabana::TeamOpTag()); - Cabana::SerialOpTag()); - Kokkos::fence(); + auto cabana_verlet_list = cell_structure.get_cabana_verlet_list(); + + // cabana_verlet_list.get_variance_max_counts(); + Kokkos::RangePolicy policy(0, unique_particles.size()); + Cabana::neighbor_parallel_for(policy, forces_kernel, cabana_verlet_list, + Cabana::FirstNeighborsTag(), + // Cabana::TeamOpTag()); + Cabana::SerialOpTag()); + Kokkos::fence(); #ifdef CALIPER - CALI_MARK_END("Cabana - calc Force"); + CALI_MARK_END("Cabana - calc Force"); #endif - } #ifdef CALIPER CALI_MARK_BEGIN("Cabana - reduction Forces"); #endif // Force and Torque reduction - Kokkos::RangePolicy policy(0, particle_storage.size()); + //Kokkos::RangePolicy policy(0, unique_particles.size()); Kokkos::parallel_for("reduction", policy, [&local_force, #ifdef ROTATION @@ -397,10 +281,12 @@ void cabana_short_range( tz += local_torque(i, tid, 2); #endif } - auto &p = unique_particles.at(i); - p->force() += Utils::Vector3d{fx, fy, fz}; + //auto &p = unique_particles.at(i); + //p->force() += Utils::Vector3d{fx, fy, fz}; + unique_particles.at(i)->force() += Utils::Vector3d{fx, fy, fz}; #ifdef ROTATION - p->torque() += Utils::Vector3d{tx, ty, tz}; + //p->torque() += Utils::Vector3d{tx, ty, tz}; + unique_particles.at(i)->torque() += Utils::Vector3d{tx, ty, tz}; #endif }); Kokkos::fence(); From aac680c03b9c2bfd56f5c4920fe112c1461e2e76 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 1 Aug 2025 18:03:32 +0200 Subject: [PATCH 69/94] Formatting --- src/core/aosoa_pack.hpp | 2 +- src/core/cell_system/CellStructure.cpp | 29 ++++---- src/core/cell_system/CellStructure.hpp | 98 +++++++++++++------------- src/core/custom_verlet_list.hpp | 6 +- src/core/forces.cpp | 30 ++++---- src/core/forces_cabana.hpp | 10 +-- src/core/integrate.cpp | 2 +- src/core/short_range_cabana.hpp | 76 ++++++++++---------- 8 files changed, 131 insertions(+), 122 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index 79f87fab229..54b74f78a8a 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -23,7 +23,7 @@ #include -//const int vector_length = 1; +// const int vector_length = 1; using data_types = Cabana::MemberTypes; //, bool>; using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index ce6a7befe44..9445129f2de 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -62,12 +62,12 @@ #ifdef SHARED_MEMORY_PARALLELISM -//using memory_space = Kokkos::HostSpace; -//using execution_space = Kokkos::DefaultExecutionSpace; +// using memory_space = Kokkos::HostSpace; +// using execution_space = Kokkos::DefaultExecutionSpace; -//using ListAlgorithm = Cabana::HalfNeighborTag; -//using ListType = Cabana::CustomVerletList; +// using ListAlgorithm = Cabana::HalfNeighborTag; +// using ListType = Cabana::CustomVerletList; CellStructure::~CellStructure() { if (m_cabana_data) { @@ -133,20 +133,21 @@ void CellStructure::reset_cabana_data() { } } -void CellStructure::rebuild_local_properties(const std::size_t num_part, const std::size_t num_threads, const double pair_cutoff) { - m_local_force = std::make_unique> - ("local_force", num_part, num_threads); +void CellStructure::rebuild_local_properties(const std::size_t num_part, + const std::size_t num_threads, + const double pair_cutoff) { + m_local_force = + std::make_unique("local_force", num_part, num_threads); #ifdef ROTATION - m_local_torque = std::make_unique> - ("local_torque", num_part, num_threads); + m_local_torque = + std::make_unique("local_torque", num_part, num_threads); #endif #ifdef NPT - m_local_virial = std::make_unique> - ("local_virial", num_threads); + m_local_virial = + std::make_unique("local_virial", num_threads); #endif m_particle_storage = - std::make_unique> - ("particles", num_part); + std::make_unique("particles", num_part); (*m_particle_storage).resize(num_part); // particle properties are defined in aosoa_pack.hpp m_aosoa = std::make_unique(*m_particle_storage); diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 4abbe36c783..d9620a4c99b 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -52,8 +52,8 @@ #include #include #include -#include #include +#include #include #ifdef CALIPER @@ -63,25 +63,21 @@ // forward declaration to not have to import cabana #ifdef SHARED_MEMORY_PARALLELISM namespace Kokkos { - template - class View; - class HostSpace; - class LayoutRight; - template - class MemoryTraits; -} +template class View; +class HostSpace; +class LayoutRight; +template class MemoryTraits; +} // namespace Kokkos namespace Cabana { - class HalfNeighborTag; - class VerletLayout2D; - class TeamVectorOpTag; - template - class CustomVerletList; - template - struct MemberTypes; - template - class AoSoA; -} +class HalfNeighborTag; +class VerletLayout2D; +class TeamVectorOpTag; +template +class CustomVerletList; +template struct MemberTypes; +template +class AoSoA; +} // namespace Cabana class CabanaData; struct AoSoA_pack; // To construct AoSoA, vector_length is defined HERE. @@ -193,16 +189,20 @@ struct CellStructure : public System::Leaf { bool m_verlet_skin_set = false; double m_verlet_reuse = 0.; #ifdef SHARED_MEMORY_PARALLELISM - std::unique_ptr> m_local_force; + using ForceType = Kokkos::View; + std::unique_ptr m_local_force; #ifdef ROTATION - std::unique_ptr> m_local_torque; + std::unique_ptr m_local_torque; #endif #ifdef NPT - std::unique_ptr> m_local_virial; + using VirialType = Kokkos::View; + std::unique_ptr m_local_virial; #endif - using data_types = Cabana::MemberTypes; //, bool>; - using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; - using AoSoAType = Cabana::AoSoA>; + using data_types = + Cabana::MemberTypes; + using memory_space = Kokkos::HostSpace; + using AoSoAType = Cabana::AoSoA>; std::unique_ptr m_particle_storage; /** particle properties for Cabana defined in aosoa_pack.hpp */ std::unique_ptr m_aosoa; @@ -210,8 +210,9 @@ struct CellStructure : public System::Leaf { std::vector m_unique_particles; using ListAlgorithm = Cabana::HalfNeighborTag; - using ListType = Cabana::CustomVerletList; + using ListType = + Cabana::CustomVerletList; std::unique_ptr m_cabana_verlet_list; #endif @@ -717,20 +718,19 @@ struct CellStructure : public System::Leaf { int m_max_id = 0; inline int estimate_max_counts(const double pair_cutoff, - const int number_of_unique_particles) { - //std::cout << "estimate_max_counts:" << pair_cutoff << " " -// << max_prefactor << std::endl; + const int number_of_unique_particles) { + // std::cout << "estimate_max_counts:" << pair_cutoff << " " + // << max_prefactor << std::endl; int max_counts; if (not std::isinf(pair_cutoff)) { - max_counts = - static_cast(std::ceil(max_prefactor * - pair_cutoff * pair_cutoff * pair_cutoff)); - int threshold_num = 16; //8; + max_counts = static_cast( + std::ceil(max_prefactor * pair_cutoff * pair_cutoff * pair_cutoff)); + int threshold_num = 16; // 8; #ifdef COLLISION_DETECTION threshold_num = 64; #endif if (max_counts < threshold_num) { - max_counts = std::min(threshold_num, number_of_unique_particles); + max_counts = std::min(threshold_num, number_of_unique_particles); } } else { max_counts = number_of_unique_particles; @@ -764,45 +764,45 @@ struct CellStructure : public System::Leaf { int get_max_id() { return m_max_id; } - void rebuild_local_properties(std::size_t num_part, std::size_t num_threads, double pair_cutoff); + void rebuild_local_properties(std::size_t num_part, std::size_t num_threads, + double pair_cutoff); void reset_local_properties(); - Kokkos::View& get_local_force() { return *m_local_force; } + ForceType &get_local_force() { return *m_local_force; } #ifdef ROTATION - Kokkos::View& get_local_torque() { return *m_local_torque; } + ForceType &get_local_torque() { return *m_local_torque; } #endif #ifdef NPT - Kokkos::View& get_local_virial() { return *m_local_virial; } + VirialType &get_local_virial() { return *m_local_virial; } #endif - AoSoA_pack& get_aosoa_data() { return *m_aosoa; }; - ListType& get_cabana_verlet_list() { return *m_cabana_verlet_list; }; - std::vector& get_unique_particles() { return m_unique_particles; } + AoSoA_pack &get_aosoa_data() { return *m_aosoa; }; + ListType &get_cabana_verlet_list() { return *m_cabana_verlet_list; }; + std::vector &get_unique_particles() { return m_unique_particles; } inline void set_index_map(ParticleRange const &particles, - ParticleRange const &ghost_particles, - int &index) { + ParticleRange const &ghost_particles, int &index) { m_unique_particles.clear(); m_max_id = 0; std::unordered_set registered_index{}; for (auto &p : particles) { if (p.id() > m_max_id) - m_max_id = p.id(); + m_max_id = p.id(); m_unique_particles.emplace_back(&p); index++; } for (auto &p : ghost_particles) { if (not get_local_particle(p.id())) { - continue; + continue; } if (not get_local_particle(p.id())->is_ghost()) { - continue; + continue; } if (registered_index.contains(p.id())) { - continue; + continue; } if (p.id() > m_max_id) - m_max_id = p.id(); + m_max_id = p.id(); registered_index.insert(p.id()); m_unique_particles.emplace_back(&p); index++; diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 856fbc40a52..6938feb1457 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -115,7 +115,7 @@ class CustomVerletList counts(pid) += 1; } - // Sorting a neighbor + // Sorting a neighbor KOKKOS_INLINE_FUNCTION void sortNeighbors() { Kokkos::parallel_for( @@ -123,8 +123,8 @@ class CustomVerletList Kokkos::RangePolicy(0, counts.size()), [&](const int i) { const int count = counts(i); - int* ptr = &neighbors(i, 0); - std::sort(ptr, ptr + count); + int *ptr = &neighbors(i, 0); + std::sort(ptr, ptr + count); }); Kokkos::fence(); } diff --git a/src/core/forces.cpp b/src/core/forces.cpp index d986d79cd36..ae8aee15c2c 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -193,14 +193,18 @@ void System::System::calculate_forces() { }; #ifdef SHARED_MEMORY_PARALLELISM - auto const &verlet_criterion = - VerletCriterion<>{*this, cell_structure->get_verlet_skin(), - get_interaction_range(), coulomb_cutoff, dipole_cutoff, - collision_detection_cutoff}; - update_cabana_state(*cell_structure, particles, cell_structure->ghost_particles(), - verlet_criterion, get_interaction_range()); + auto const &verlet_criterion = + VerletCriterion<>{*this, + cell_structure->get_verlet_skin(), + get_interaction_range(), + coulomb_cutoff, + dipole_cutoff, + collision_detection_cutoff}; + update_cabana_state(*cell_structure, particles, + cell_structure->ghost_particles(), verlet_criterion, + get_interaction_range()); #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) auto unique_particles = cell_structure->get_unique_particles(); #endif auto local_force = cell_structure->get_local_force(); @@ -221,7 +225,7 @@ void System::System::calculate_forces() { #endif *box_geo, #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) + defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) unique_particles, #endif local_force, @@ -233,13 +237,13 @@ void System::System::calculate_forces() { #endif aosoa); - cabana_short_range( - bond_kernel, first_neighbor_kernel, + cabana_short_range(bond_kernel, first_neighbor_kernel, #ifdef COLLISION_DETECTION - collision_detection, + collision_detection, #endif - *cell_structure, get_interaction_range(), bonded_ias->maximal_cutoff(), - particles, cell_structure->ghost_particles(), verlet_criterion); + *cell_structure, get_interaction_range(), + bonded_ias->maximal_cutoff(), particles, + cell_structure->ghost_particles(), verlet_criterion); #else auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index a55b6b95ce8..fa9adbb5305 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -88,16 +88,16 @@ struct ForcesKernel { box_geo(box_geo_), #if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) - unique_particles(unique_particles_), + unique_particles(unique_particles_), #endif - local_force(local_force_), + local_force(local_force_), #ifdef ROTATION - local_torque(local_torque_), + local_torque(local_torque_), #endif #ifdef NPT - local_virial(local_virial_), + local_virial(local_virial_), #endif - aosoa(aosoa_) { + aosoa(aosoa_) { } __attribute__((always_inline)) KOKKOS_INLINE_FUNCTION void diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index f52884572e7..7f38ae59ac1 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -527,7 +527,7 @@ int System::System::integrate(int n_steps, int reuse_forces) { ek_active = ek.is_ready_for_propagation(); #ifdef SHARED_MEMORY_PARALLELISM // cell_structure->set_steepest_descent_flag(false); - cell_structure->set_max_prefactor(5); + cell_structure->set_max_prefactor(5); #endif } #ifdef SHARED_MEMORY_PARALLELISM diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 1dc544a74b7..cd582e1908d 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -30,14 +30,15 @@ #ifdef SHARED_MEMORY_PARALLELISM #include "aosoa_pack.hpp" -//#include "cabana_data.hpp" +// #include "cabana_data.hpp" #include "custom_verlet_list.hpp" #include "forces_cabana.hpp" #include #include #include -inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa) { +inline void write_particle(Particle const &p, int const &id, + AoSoA_pack &aosoa) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); @@ -51,11 +52,9 @@ using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; template -__attribute__((always_inline)) inline void -construct_verlet_list(CellStructure &cell_structure, - VerletCriterion const &verlet_criterion, - Kokkos::View const &id_to_index, - const int max_id) { +__attribute__((always_inline)) inline void construct_verlet_list( + CellStructure &cell_structure, VerletCriterion const &verlet_criterion, + Kokkos::View const &id_to_index, const int max_id) { auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); auto const distance_function = detail::MinimalImageDistance{ @@ -63,7 +62,7 @@ construct_verlet_list(CellStructure &cell_structure, auto verlet_list = cell_structure.get_cabana_verlet_list(); auto intra_kernel = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](const int i) { + &id_to_index, &verlet_list, max_id](const int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto &p1 = *it; @@ -87,7 +86,7 @@ construct_verlet_list(CellStructure &cell_structure, }; auto inter_kernel = [&cells, &distance_function, &verlet_criterion, - &id_to_index, &verlet_list, max_id](const int i) { + &id_to_index, &verlet_list, max_id](const int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto const &p1 = *it; @@ -123,12 +122,10 @@ construct_verlet_list(CellStructure &cell_structure, } template -__attribute__((always_inline)) inline void -update_cabana_state(CellStructure &cell_structure, - ParticleRange const &particles, - ParticleRange const &ghost_particles, - VerletCriterion const &verlet_criterion, - double const pair_cutoff) { +__attribute__((always_inline)) inline void update_cabana_state( + CellStructure &cell_structure, ParticleRange const &particles, + ParticleRange const &ghost_particles, + VerletCriterion const &verlet_criterion, double const pair_cutoff) { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Index map"); #endif @@ -137,14 +134,17 @@ update_cabana_state(CellStructure &cell_structure, int number_of_unique_particles = 0; - bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); + bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or + (not cell_structure.use_verlet_list); // std::cout << "rebuild:" << rebuild << std::endl; if (rebuild) { // If we have to rebuild, we need to count the particles - cell_structure.set_index_map(particles, ghost_particles, number_of_unique_particles); + cell_structure.set_index_map(particles, ghost_particles, + number_of_unique_particles); // Create essential variable for MD - cell_structure.rebuild_local_properties(number_of_unique_particles, num_threads, pair_cutoff); + cell_structure.rebuild_local_properties(number_of_unique_particles, + num_threads, pair_cutoff); } else { // If we do not rebuild we can use the saved map number_of_unique_particles = cell_structure.get_unique_particles().size(); @@ -166,16 +166,16 @@ update_cabana_state(CellStructure &cell_structure, // Fill particle storage // =================================================== Kokkos::View id_to_index( - Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); + Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); Kokkos::deep_copy(id_to_index, -1); using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for( - "AoSoA write", policy_type(0, number_of_unique_particles), - [&unique_particles, &aosoa, &id_to_index](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, aosoa); - id_to_index(unique_particles.at(p_id)->id()) = p_id; - }); + "AoSoA write", policy_type(0, number_of_unique_particles), + [&unique_particles, &aosoa, &id_to_index](const int p_id) { + write_particle(*unique_particles.at(p_id), p_id, aosoa); + id_to_index(unique_particles.at(p_id)->id()) = p_id; + }); Kokkos::fence(); #ifdef CALIPER CALI_MARK_END("Cabana - Allocation"); @@ -190,7 +190,8 @@ update_cabana_state(CellStructure &cell_structure, #ifdef CALIPER CALI_MARK_BEGIN("Cabana - Verlet List"); #endif - construct_verlet_list(cell_structure, verlet_criterion, id_to_index, max_id); + construct_verlet_list(cell_structure, verlet_criterion, id_to_index, + max_id); cell_structure.mark_rebuild_cabana_verlet_list_as_UpToDate(); #ifdef CALIPER CALI_MARK_END("Cabana - Verlet List"); @@ -199,7 +200,8 @@ update_cabana_state(CellStructure &cell_structure, } } -template +template void cabana_short_range( BondKernel const &bond_kernel, PairKernel const &forces_kernel, #ifdef COLLISION_DETECTION @@ -212,7 +214,7 @@ void cabana_short_range( CALI_CXX_MARK_FUNCTION; #endif - int num_threads = execution_space().concurrency(); + int num_threads = execution_space().concurrency(); #ifdef CALIPER CALI_MARK_BEGIN("Espresso - Bond Kernel"); @@ -244,9 +246,9 @@ void cabana_short_range( // cabana_verlet_list.get_variance_max_counts(); Kokkos::RangePolicy policy(0, unique_particles.size()); Cabana::neighbor_parallel_for(policy, forces_kernel, cabana_verlet_list, - Cabana::FirstNeighborsTag(), - // Cabana::TeamOpTag()); - Cabana::SerialOpTag()); + Cabana::FirstNeighborsTag(), + // Cabana::TeamOpTag()); + Cabana::SerialOpTag()); Kokkos::fence(); #ifdef CALIPER CALI_MARK_END("Cabana - calc Force"); @@ -256,7 +258,7 @@ void cabana_short_range( CALI_MARK_BEGIN("Cabana - reduction Forces"); #endif // Force and Torque reduction - //Kokkos::RangePolicy policy(0, unique_particles.size()); + // Kokkos::RangePolicy policy(0, unique_particles.size()); Kokkos::parallel_for("reduction", policy, [&local_force, #ifdef ROTATION @@ -281,12 +283,14 @@ void cabana_short_range( tz += local_torque(i, tid, 2); #endif } - //auto &p = unique_particles.at(i); - //p->force() += Utils::Vector3d{fx, fy, fz}; - unique_particles.at(i)->force() += Utils::Vector3d{fx, fy, fz}; + // auto &p = unique_particles.at(i); + // p->force() += Utils::Vector3d{fx, fy, fz}; + unique_particles.at(i)->force() += + Utils::Vector3d{fx, fy, fz}; #ifdef ROTATION - //p->torque() += Utils::Vector3d{tx, ty, tz}; - unique_particles.at(i)->torque() += Utils::Vector3d{tx, ty, tz}; + // p->torque() += Utils::Vector3d{tx, ty, tz}; + unique_particles.at(i)->torque() += + Utils::Vector3d{tx, ty, tz}; #endif }); Kokkos::fence(); From 1774c9a6df59c9b4d3fe9317ed86fb84d023fa0a Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 1 Aug 2025 18:05:17 +0200 Subject: [PATCH 70/94] Formatting --- src/core/cell_system/CellStructure.cpp | 6 ++---- src/core/cell_system/CellStructure.hpp | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 9445129f2de..1701acc094e 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -143,11 +143,9 @@ void CellStructure::rebuild_local_properties(const std::size_t num_part, std::make_unique("local_torque", num_part, num_threads); #endif #ifdef NPT - m_local_virial = - std::make_unique("local_virial", num_threads); + m_local_virial = std::make_unique("local_virial", num_threads); #endif - m_particle_storage = - std::make_unique("particles", num_part); + m_particle_storage = std::make_unique("particles", num_part); (*m_particle_storage).resize(num_part); // particle properties are defined in aosoa_pack.hpp m_aosoa = std::make_unique(*m_particle_storage); diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index d9620a4c99b..77056be01f5 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -198,8 +198,7 @@ struct CellStructure : public System::Leaf { using VirialType = Kokkos::View; std::unique_ptr m_local_virial; #endif - using data_types = - Cabana::MemberTypes; + using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; using AoSoAType = Cabana::AoSoA>; From 30b9b6775515e3be134e22182641c614dce1a109 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 1 Aug 2025 19:19:17 +0200 Subject: [PATCH 71/94] Fixed warnings --- src/core/cell_system/CellStructure.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 77056be01f5..a5c1e465880 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -65,12 +65,12 @@ namespace Kokkos { template class View; class HostSpace; -class LayoutRight; -template class MemoryTraits; +struct LayoutRight; +template struct MemoryTraits; } // namespace Kokkos namespace Cabana { class HalfNeighborTag; -class VerletLayout2D; +struct VerletLayout2D; class TeamVectorOpTag; template class CustomVerletList; @@ -712,8 +712,8 @@ struct CellStructure : public System::Leaf { private: std::unique_ptr m_cabana_data; // bool steepest_descent_flag = true; - std::size_t max_prefactor = 8; - std::size_t max_counts = -1; + int max_prefactor = 8; + int max_counts = -1; int m_max_id = 0; inline int estimate_max_counts(const double pair_cutoff, From f51269f5b8370894fde922ce6cf96e83b7eb45eb Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 1 Aug 2025 19:41:13 +0200 Subject: [PATCH 72/94] Fixed warnings --- src/core/cell_system/CellStructure.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index a5c1e465880..0c88bb88b9c 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -755,8 +755,7 @@ struct CellStructure : public System::Leaf { // void set_steepest_descent_flag(bool flag) { steepest_descent_flag = flag; } // bool get_steepest_descent_flag() { return steepest_descent_flag; } - void set_max_prefactor(std::size_t value) { max_prefactor = value; } - std::size_t get_max_prefactor() { return max_prefactor; } + void set_max_prefactor(int value) { max_prefactor = value; } void set_max_counts(std::size_t value) { max_counts = value; } std::size_t get_max_counts() { return max_counts; } From e9617f8f35ead5a1fb7b82e5858f3a597d17e026 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Fri, 1 Aug 2025 20:44:51 +0200 Subject: [PATCH 73/94] Fixed warnings --- src/core/cell_system/CellStructure.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 0c88bb88b9c..58962198156 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -757,8 +757,8 @@ struct CellStructure : public System::Leaf { void set_max_prefactor(int value) { max_prefactor = value; } - void set_max_counts(std::size_t value) { max_counts = value; } - std::size_t get_max_counts() { return max_counts; } + void set_max_counts(int value) { max_counts = value; } + int get_max_counts() { return max_counts; } int get_max_id() { return m_max_id; } From d5ad3147a2be340ace0de14271ca057a9c33c1c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Tue, 5 Aug 2025 13:42:50 +0200 Subject: [PATCH 74/94] Sort out ifdefs --- src/core/forces.cpp | 13 +++++-------- src/core/forces_cabana.hpp | 9 +++------ src/core/forces_inline.hpp | 28 ++++++++++++---------------- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/core/forces.cpp b/src/core/forces.cpp index d1332deef08..aaa6beb28e8 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -234,8 +234,7 @@ void System::System::calculate_forces() { update_cabana_state(*cell_structure, particles, cell_structure->ghost_particles(), verlet_criterion, get_interaction_range()); -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) +#if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) auto unique_particles = cell_structure->get_unique_particles(); #endif auto local_force = cell_structure->get_local_force(); @@ -249,14 +248,12 @@ void System::System::calculate_forces() { ForcesKernel first_neighbor_kernel( *bonded_ias, *nonbonded_ias, get_ptr(coulomb_kernel), -#if defined(THOLE) or defined(ELECTROSTATICS) or defined(P3M) or \ - defined(DPD) or defined(DIPOLES) or defined(NPT) +#if defined(LONG_RANGE_KERNELS) get_ptr(dipoles_kernel), get_ptr(elc_kernel), get_ptr(coulomb_u_kernel), *thermostat, #endif *box_geo, -#if defined(EXCLUSIONS) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) or defined(NPT) +#if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) unique_particles, #endif local_force, @@ -275,7 +272,7 @@ void System::System::calculate_forces() { *cell_structure, get_interaction_range(), bonded_ias->maximal_cutoff(), particles, cell_structure->ghost_particles(), verlet_criterion); -#else +#else // SHARED_MEMORY_PARALLELISM auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), dipoles_kernel_ptr = get_ptr(dipoles_kernel), @@ -307,7 +304,7 @@ void System::System::calculate_forces() { dipole_cutoff, collision_detection_cutoff}); -#endif +#endif // SHARED_MEMORY_PARALLELISM constraints->add_forces(particles, get_sim_time()); oif_global->calculate_forces(); diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index b738306e538..af27d2f624a 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -27,11 +27,8 @@ #include "aosoa_pack.hpp" #include "forces_inline.hpp" -#include -#if defined(ELECTROSTATICS) or defined(DIPOLES) or defined(DPD) or defined(NPT) -#define LONG_RANGE_KERNELS -#endif +#include struct ForcesKernel { [[maybe_unused]] const BondedInteractionsMap &bonded_ias; @@ -128,8 +125,8 @@ struct ForcesKernel { auto constexpr do_nonbonded_flag = true; #endif - add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, - do_nonbonded_flag, coulomb_kernel); + add_non_bonded_pair_without_p(pf, d, dist, q1q2, ia_params, + do_nonbonded_flag, coulomb_kernel); #if defined(LONG_RANGE_KERNELS) add_non_bonded_pair_force_with_p(p1, p2, pf, diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 39297366654..f1342d7c423 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -29,6 +29,10 @@ #include "forces.hpp" +#if defined(ELECTROSTATICS) or defined(DIPOLES) or defined(DPD) or defined(NPT) +#define LONG_RANGE_KERNELS +#endif + #include "BoxGeometry.hpp" #include "actor/visitors.hpp" #include "bond_breakage/bond_breakage.hpp" @@ -174,7 +178,7 @@ inline ParticleForce calc_opposing_force(ParticleForce const &pf, /** * For the interaction which need NO particle information */ -inline void add_non_bonded_pair_withot_p( +inline void add_non_bonded_pair_without_p( ParticleForce &pf, Utils::Vector3d const &d, double dist, double q1q2, IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel) { @@ -214,7 +218,7 @@ inline void add_non_bonded_pair_withot_p( */ inline void add_non_bonded_pair_force_with_p( Particle &p1, Particle &p2, ParticleForce &pf, -#ifdef NPT +#if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) Utils::Vector3d &virial, #endif Utils::Vector3d const &d, double dist, double dist2, double q1q2, @@ -312,13 +316,6 @@ inline void add_non_bonded_pair_force_with_p( // return std::pair{pf, virial}; } -#if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) -using ReturnType = std::pair; -#elif defined(SHARED_MEMORY_PARALLELISM) -using ReturnType = ParticleForce; -#else -using ReturnType = void; -#endif /** Calculate non-bonded forces between a pair of particles and update their * forces and torques. * @param[in,out] p1 particle 1. @@ -336,7 +333,7 @@ using ReturnType = void; * @param[in] elc_kernel ELC force correction kernel. * @param[in] coulomb_u_kernel Coulomb energy kernel. */ -inline ReturnType add_non_bonded_pair_force( +inline auto add_non_bonded_pair_force( Particle &p1, Particle &p2, Utils::Vector3d const &d, double dist, double dist2, double q1q2, IA_parameters const &ia_params, Thermostat::Thermostat const &thermostat, BoxGeometry const &box_geo, @@ -352,16 +349,15 @@ inline ReturnType add_non_bonded_pair_force( #endif #ifdef EXCLUSIONS - bool do_nonbonded_flag = do_nonbonded(p1, p2); + auto const do_nonbonded_flag = do_nonbonded(p1, p2); #else - bool do_nonbonded_flag = true; + auto constexpr do_nonbonded_flag = true; #endif - add_non_bonded_pair_withot_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, - coulomb_kernel); + add_non_bonded_pair_without_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, + coulomb_kernel); -#if defined(NPT) or defined(THOLE) or defined(ELECTROSTATICS) or \ - defined(P3M) or defined(DPD) or defined(DIPOLES) +#if defined(LONG_RANGE_KERNELS) add_non_bonded_pair_force_with_p( p1, p2, pf, #if defined(NPT) and defined(SHARED_MEMORY_PARALLELISM) From a0c0ba77c9d308fd1dc07142cf3ac87a561ea71e Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 5 Aug 2025 18:05:08 +0200 Subject: [PATCH 75/94] Refactoring --- src/core/aosoa_pack.hpp | 15 ++-- src/core/cell_system/CellStructure.cpp | 57 +++++++++++-- src/core/cell_system/CellStructure.hpp | 43 ++++++---- src/core/forces.cpp | 79 ++++++++++++++++-- src/core/forces_cabana.hpp | 12 +-- src/core/short_range_cabana.hpp | 109 +++---------------------- 6 files changed, 175 insertions(+), 140 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index 54b74f78a8a..a6611334ae3 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -21,23 +21,20 @@ #ifdef SHARED_MEMORY_PARALLELISM +#include "cell_system/CellStructure.hpp" #include -// const int vector_length = 1; -using data_types = Cabana::MemberTypes; //, bool>; -using memory_space = Kokkos::HostSpace; // Kokkos::SharedSpace; using execution_space = Kokkos::DefaultExecutionSpace; -using AoSoA_type = Cabana::AoSoA; struct AoSoA_pack { - AoSoA_type::member_slice_type<0> position; - AoSoA_type::member_slice_type<1> charge; - AoSoA_type::member_slice_type<2> id; - AoSoA_type::member_slice_type<3> type; + AoSoAType::member_slice_type<0> position; + AoSoAType::member_slice_type<1> charge; + AoSoAType::member_slice_type<2> id; + AoSoAType::member_slice_type<3> type; AoSoA_pack() = default; - AoSoA_pack(AoSoA_type &aosoa) + AoSoA_pack(AoSoAType &aosoa) : position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)) {} }; diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 09a0f7c9192..2514105d679 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -32,6 +32,7 @@ #include "cell_system/CellStructureType.hpp" #include "communication.hpp" #include "lees_edwards/lees_edwards.hpp" +#include "particle_enumeration.hpp" #include "particle_reduction.hpp" #include "system/System.hpp" @@ -63,16 +64,12 @@ #include #include #endif +#ifdef CALIPER +#include "caliper/cali.h" +#endif #ifdef SHARED_MEMORY_PARALLELISM -// using memory_space = Kokkos::HostSpace; -// using execution_space = Kokkos::DefaultExecutionSpace; - -// using ListAlgorithm = Cabana::HalfNeighborTag; -// using ListType = Cabana::CustomVerletList; - CellStructure::~CellStructure() { if (m_cabana_data) { m_cabana_data.reset(); @@ -163,10 +160,56 @@ void CellStructure::reset_local_properties() { #ifdef ROTATION Kokkos::deep_copy(get_local_torque(), 0); #endif + /* + Kokkos::parallel_for(get_local_force().extent(0), [&](int i) { + for (int j = 0; j < get_local_force().extent(1); j++) { + for (int k : {0, 1, 2}) { + get_local_force()(i, j, k) = 0.; +#ifdef ROTATION + get_local_torque()(i, j, k) = 0.; +#endif + } + } + });*/ #ifdef NPT Kokkos::deep_copy(get_local_virial(), 0); #endif } + +void CellStructure::set_index_map() { +#ifdef CALIPER + CALI_CXX_MARK_FUNCTION; +#endif + m_unique_particles.clear(); + m_unique_particles.resize(count_local_particles()); + std::unordered_set registered_index{}; + using execution_space = Kokkos::DefaultExecutionSpace; + int n_threads = execution_space().concurrency(); + std::vector max_ids(n_threads); + enumerate_local_particles(*this, [&](int index, Particle &p) { + m_unique_particles[index] = &p; + const int thread_num = omp_get_thread_num(); + max_ids[thread_num] = std::max(p.id(), max_ids[thread_num]); + }); + int max_id = *(std::max_element(max_ids.begin(), max_ids.end())); + for (auto &p : ghost_particles()) { + const Particle *local_particle = get_local_particle(p.id()); + if (not local_particle) { + continue; + } + if (not local_particle->is_ghost()) { + continue; + } + if (registered_index.contains(p.id())) { + continue; + } + registered_index.insert(p.id()); + m_unique_particles.emplace_back(&p); + max_id = std::max(p.id(), max_id); + } + registered_index.clear(); + m_cached_max_local_particle_id = max_id; +} #endif CellStructure::CellStructure(BoxGeometry const &box) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 3fc960bb5f3..af96b389b86 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -82,6 +82,17 @@ class CabanaData; struct AoSoA_pack; // To construct AoSoA, vector_length is defined HERE. const int vector_length = 1; + +using ForceType = Kokkos::View; +using VirialType = Kokkos::View; +using data_types = Cabana::MemberTypes; +using memory_space = Kokkos::HostSpace; +using AoSoAType = Cabana::AoSoA>; +using ListAlgorithm = Cabana::HalfNeighborTag; +using ListType = + Cabana::CustomVerletList; #endif template @@ -189,29 +200,21 @@ struct CellStructure : public System::Leaf { bool m_verlet_skin_set = false; double m_verlet_reuse = 0.; #ifdef SHARED_MEMORY_PARALLELISM - using ForceType = Kokkos::View; + int m_cached_max_local_particle_id; + std::unique_ptr m_local_force; #ifdef ROTATION std::unique_ptr m_local_torque; #endif #ifdef NPT - using VirialType = Kokkos::View; std::unique_ptr m_local_virial; #endif - using data_types = Cabana::MemberTypes; - using memory_space = Kokkos::HostSpace; - using AoSoAType = Cabana::AoSoA>; std::unique_ptr m_particle_storage; /** particle properties for Cabana defined in aosoa_pack.hpp */ std::unique_ptr m_aosoa; /** The local id-to-index for aosoa data */ std::vector m_unique_particles; - using ListAlgorithm = Cabana::HalfNeighborTag; - using ListType = - Cabana::CustomVerletList; std::unique_ptr m_cabana_verlet_list; #endif @@ -340,6 +343,14 @@ struct CellStructure : public System::Leaf { return Cells::particles(decomposition().ghost_cells()); } + int count_local_particles() const { + int count = 0; + for (auto const &cell : m_decomposition->local_cells()) { + count += cell->particles().size(); + } + return count; + } + /** @brief whether to use parallel version of @ref for_each_local_particle */ bool use_parallel_for_each_local_particle() const { #ifdef SHARED_MEMORY_PARALLELISM @@ -442,6 +453,11 @@ struct CellStructure : public System::Leaf { * this node, or -1 if there are no particles on this node. */ int get_max_local_particle_id() const; +#ifdef SHARED_MEMORY_PARALLELISM + int get_cached_max_local_particle_id() const { + return m_cached_max_local_particle_id; + }; +#endif /** * @brief Remove all particles from the cell system. @@ -751,7 +767,7 @@ struct CellStructure : public System::Leaf { void set_max_prefactor(int value) { max_prefactor = value; } void set_max_counts(int value) { max_counts = value; } - int get_max_counts() { return max_counts; } + int get_max_counts() const { return max_counts; } int get_max_id() { return m_max_id; } @@ -770,8 +786,9 @@ struct CellStructure : public System::Leaf { ListType &get_cabana_verlet_list() { return *m_cabana_verlet_list; }; std::vector &get_unique_particles() { return m_unique_particles; } + void set_index_map(); inline void set_index_map(ParticleRange const &particles, - ParticleRange const &ghost_particles, int &index) { + ParticleRange const &ghost_particles) { m_unique_particles.clear(); m_max_id = 0; std::unordered_set registered_index{}; @@ -779,7 +796,6 @@ struct CellStructure : public System::Leaf { if (p.id() > m_max_id) m_max_id = p.id(); m_unique_particles.emplace_back(&p); - index++; } for (auto &p : ghost_particles) { @@ -796,7 +812,6 @@ struct CellStructure : public System::Leaf { m_max_id = p.id(); registered_index.insert(p.id()); m_unique_particles.emplace_back(&p); - index++; } registered_index.clear(); } diff --git a/src/core/forces.cpp b/src/core/forces.cpp index aaa6beb28e8..de7e52f8cbb 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -234,9 +234,7 @@ void System::System::calculate_forces() { update_cabana_state(*cell_structure, particles, cell_structure->ghost_particles(), verlet_criterion, get_interaction_range()); -#if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) auto unique_particles = cell_structure->get_unique_particles(); -#endif auto local_force = cell_structure->get_local_force(); #ifdef ROTATION auto local_torque = cell_structure->get_local_torque(); @@ -266,12 +264,83 @@ void System::System::calculate_forces() { aosoa); cabana_short_range(bond_kernel, first_neighbor_kernel, -#ifdef COLLISION_DETECTION - collision_detection, -#endif *cell_structure, get_interaction_range(), bonded_ias->maximal_cutoff(), particles, cell_structure->ghost_particles(), verlet_criterion); +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - reduction Forces"); +#endif + // Force and Torque reduction + int num_threads = execution_space().concurrency(); + Kokkos::RangePolicy policy(0, unique_particles.size()); + Kokkos::parallel_for("reduction", policy, + [&local_force, +#ifdef ROTATION + &local_torque, +#endif + &unique_particles, num_threads](const int i) { + double fx = 0.; + double fy = 0.; + double fz = 0.; +#ifdef ROTATION + double tx = 0.; + double ty = 0.; + double tz = 0.; +#endif + for (int tid = 0; tid < num_threads; ++tid) { + fx += local_force(i, tid, 0); + fy += local_force(i, tid, 1); + fz += local_force(i, tid, 2); +#ifdef ROTATION + tx += local_torque(i, tid, 0); + ty += local_torque(i, tid, 1); + tz += local_torque(i, tid, 2); +#endif + } + // auto &p = unique_particles.at(i); + // p->force() += Utils::Vector3d{fx, fy, fz}; + unique_particles.at(i)->force() += + Utils::Vector3d{fx, fy, fz}; +#ifdef ROTATION + // p->torque() += Utils::Vector3d{tx, ty, tz}; + unique_particles.at(i)->torque() += + Utils::Vector3d{tx, ty, tz}; +#endif + }); + Kokkos::fence(); + +#ifdef NPT + double vx = 0.; + double vy = 0.; + double vz = 0.; + for (int tid = 0; tid < num_threads; ++tid) { + vx += local_virial(tid, 0); + vy += local_virial(tid, 1); + vz += local_virial(tid, 2); + } + Utils::Vector3d virial_vec{vx, vy, vz}; + npt_add_virial_force_contribution(virial_vec); +#endif +#ifdef CALIPER + CALI_MARK_END("Cabana - reduction Forces"); +#endif + +#ifdef CALIPER + CALI_MARK_BEGIN("Cabana - Collision Detection"); +#endif +#ifdef COLLISION_DETECTION + auto collision_kernel = [&collision_detection = *collision_detection] + (Particle const &p1, Particle const &p2, Distance const &d) { + if (not collision_detection.is_off()) { + collision_detection.detect_collision(p1, p2, d.dist2); + } + }; + cell_structure->non_bonded_loop(collision_kernel, verlet_criterion); +#endif +#ifdef CALIPER + CALI_MARK_END("Cabana - Collision Detection"); +#endif + #else // SHARED_MEMORY_PARALLELISM auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index af27d2f624a..fdc51026ffd 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -44,12 +44,12 @@ struct ForcesKernel { #if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) std::vector &unique_particles; #endif - Kokkos::View local_force; + ForceType local_force; #ifdef ROTATION - Kokkos::View local_torque; + ForceType local_torque; #endif #ifdef NPT - Kokkos::View local_virial; + VirialType local_virial; #endif const AoSoA_pack &aosoa; @@ -67,12 +67,12 @@ struct ForcesKernel { #if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) std::vector &unique_particles_, #endif - Kokkos::View local_force_, + ForceType local_force_, #ifdef ROTATION - Kokkos::View local_torque_, + ForceType local_torque_, #endif #ifdef NPT - Kokkos::View local_virial_, + VirialType local_virial_, #endif const AoSoA_pack &aosoa_) : bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index cd582e1908d..1cde4c9b968 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -30,7 +30,6 @@ #ifdef SHARED_MEMORY_PARALLELISM #include "aosoa_pack.hpp" -// #include "cabana_data.hpp" #include "custom_verlet_list.hpp" #include "forces_cabana.hpp" #include @@ -48,9 +47,6 @@ inline void write_particle(Particle const &p, int const &id, } } -using ListAlgorithm = Cabana::HalfNeighborTag; -using ListType = Cabana::CustomVerletList; template __attribute__((always_inline)) inline void construct_verlet_list( CellStructure &cell_structure, VerletCriterion const &verlet_criterion, @@ -132,7 +128,7 @@ __attribute__((always_inline)) inline void update_cabana_state( // Number of threads int num_threads = execution_space().concurrency(); - int number_of_unique_particles = 0; + //int number_of_unique_particles = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); @@ -140,18 +136,20 @@ __attribute__((always_inline)) inline void update_cabana_state( if (rebuild) { // If we have to rebuild, we need to count the particles - cell_structure.set_index_map(particles, ghost_particles, - number_of_unique_particles); + // cell_structure.set_index_map(); // parallelized index_map + cell_structure.set_index_map(particles, ghost_particles); + // Create essential variable for MD - cell_structure.rebuild_local_properties(number_of_unique_particles, - num_threads, pair_cutoff); + cell_structure.rebuild_local_properties( + cell_structure.get_unique_particles().size(), num_threads, pair_cutoff); } else { // If we do not rebuild we can use the saved map - number_of_unique_particles = cell_structure.get_unique_particles().size(); + // number_of_unique_particles = cell_structure.get_unique_particles().size(); cell_structure.reset_local_properties(); } auto const unique_particles = cell_structure.get_unique_particles(); auto aosoa = cell_structure.get_aosoa_data(); + // int max_id = cell_structure.get_cached_max_local_particle_id(); int max_id = cell_structure.get_max_id(); #ifdef CALIPER @@ -171,7 +169,7 @@ __attribute__((always_inline)) inline void update_cabana_state( using policy_type = Kokkos::RangePolicy; Kokkos::parallel_for( - "AoSoA write", policy_type(0, number_of_unique_particles), + "AoSoA write", policy_type(0, unique_particles.size()), [&unique_particles, &aosoa, &id_to_index](const int p_id) { write_particle(*unique_particles.at(p_id), p_id, aosoa); id_to_index(unique_particles.at(p_id)->id()) = p_id; @@ -204,9 +202,6 @@ template void cabana_short_range( BondKernel const &bond_kernel, PairKernel const &forces_kernel, -#ifdef COLLISION_DETECTION - std::shared_ptr collision_detection, -#endif CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, ParticleRange const &particles, ParticleRange const &ghost_particles, VerletCriterion const &verlet_criterion = {}) { @@ -214,8 +209,6 @@ void cabana_short_range( CALI_CXX_MARK_FUNCTION; #endif - int num_threads = execution_space().concurrency(); - #ifdef CALIPER CALI_MARK_BEGIN("Espresso - Bond Kernel"); #endif @@ -232,19 +225,10 @@ void cabana_short_range( if (pair_cutoff > 0.) { #ifdef CALIPER CALI_MARK_BEGIN("Cabana - calc Force"); -#endif - auto unique_particles = cell_structure.get_unique_particles(); - auto local_force = cell_structure.get_local_force(); -#ifdef ROTATION - auto local_torque = cell_structure.get_local_torque(); -#endif -#ifdef NPT - auto local_virial = cell_structure.get_local_virial(); #endif auto cabana_verlet_list = cell_structure.get_cabana_verlet_list(); - // cabana_verlet_list.get_variance_max_counts(); - Kokkos::RangePolicy policy(0, unique_particles.size()); + Kokkos::RangePolicy policy(0, cell_structure.get_unique_particles().size()); Cabana::neighbor_parallel_for(policy, forces_kernel, cabana_verlet_list, Cabana::FirstNeighborsTag(), // Cabana::TeamOpTag()); @@ -253,79 +237,6 @@ void cabana_short_range( #ifdef CALIPER CALI_MARK_END("Cabana - calc Force"); #endif - -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - reduction Forces"); -#endif - // Force and Torque reduction - // Kokkos::RangePolicy policy(0, unique_particles.size()); - Kokkos::parallel_for("reduction", policy, - [&local_force, -#ifdef ROTATION - &local_torque, -#endif - &unique_particles, num_threads](const int i) { - double fx = 0.; - double fy = 0.; - double fz = 0.; -#ifdef ROTATION - double tx = 0.; - double ty = 0.; - double tz = 0.; -#endif - for (int tid = 0; tid < num_threads; ++tid) { - fx += local_force(i, tid, 0); - fy += local_force(i, tid, 1); - fz += local_force(i, tid, 2); -#ifdef ROTATION - tx += local_torque(i, tid, 0); - ty += local_torque(i, tid, 1); - tz += local_torque(i, tid, 2); -#endif - } - // auto &p = unique_particles.at(i); - // p->force() += Utils::Vector3d{fx, fy, fz}; - unique_particles.at(i)->force() += - Utils::Vector3d{fx, fy, fz}; -#ifdef ROTATION - // p->torque() += Utils::Vector3d{tx, ty, tz}; - unique_particles.at(i)->torque() += - Utils::Vector3d{tx, ty, tz}; -#endif - }); - Kokkos::fence(); - -#ifdef NPT - double vx = 0.; - double vy = 0.; - double vz = 0.; - for (int tid = 0; tid < num_threads; ++tid) { - vx += local_virial(tid, 0); - vy += local_virial(tid, 1); - vz += local_virial(tid, 2); - } - Utils::Vector3d virial_vec{vx, vy, vz}; - npt_add_virial_force_contribution(virial_vec); -#endif -#ifdef CALIPER - CALI_MARK_END("Cabana - reduction Forces"); -#endif - -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Collision Detection"); -#endif -#ifdef COLLISION_DETECTION - auto collision_kernel = [&](Particle const &p1, Particle const &p2, - Distance const &d) { - if (not collision_detection->is_off()) { - collision_detection->detect_collision(p1, p2, d.dist2); - } - }; - cell_structure.non_bonded_loop(collision_kernel, verlet_criterion); -#endif -#ifdef CALIPER - CALI_MARK_END("Cabana - Collision Detection"); -#endif } } From 3f8865825b0b3518cd1299941d85831eb07464d7 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 5 Aug 2025 18:17:13 +0200 Subject: [PATCH 76/94] Formatting --- src/core/cell_system/CellStructure.hpp | 4 +- src/core/forces.cpp | 67 +++++++++++++------------- src/core/short_range_cabana.hpp | 21 ++++---- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index af96b389b86..39cca8122be 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -88,11 +88,11 @@ using VirialType = Kokkos::View; using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; using AoSoAType = Cabana::AoSoA>; + Kokkos::MemoryTraits<0>>; using ListAlgorithm = Cabana::HalfNeighborTag; using ListType = Cabana::CustomVerletList; + Cabana::VerletLayout2D, Cabana::TeamVectorOpTag>; #endif template diff --git a/src/core/forces.cpp b/src/core/forces.cpp index de7e52f8cbb..8d509f6b84d 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -263,10 +263,10 @@ void System::System::calculate_forces() { #endif aosoa); - cabana_short_range(bond_kernel, first_neighbor_kernel, - *cell_structure, get_interaction_range(), - bonded_ias->maximal_cutoff(), particles, - cell_structure->ghost_particles(), verlet_criterion); + cabana_short_range(bond_kernel, first_neighbor_kernel, *cell_structure, + get_interaction_range(), bonded_ias->maximal_cutoff(), + particles, cell_structure->ghost_particles(), + verlet_criterion); #ifdef CALIPER CALI_MARK_BEGIN("Cabana - reduction Forces"); #endif @@ -274,39 +274,39 @@ void System::System::calculate_forces() { int num_threads = execution_space().concurrency(); Kokkos::RangePolicy policy(0, unique_particles.size()); Kokkos::parallel_for("reduction", policy, - [&local_force, + [&local_force, #ifdef ROTATION - &local_torque, + &local_torque, #endif - &unique_particles, num_threads](const int i) { - double fx = 0.; - double fy = 0.; - double fz = 0.; + &unique_particles, num_threads](const int i) { + double fx = 0.; + double fy = 0.; + double fz = 0.; #ifdef ROTATION - double tx = 0.; - double ty = 0.; - double tz = 0.; -#endif - for (int tid = 0; tid < num_threads; ++tid) { - fx += local_force(i, tid, 0); - fy += local_force(i, tid, 1); - fz += local_force(i, tid, 2); + double tx = 0.; + double ty = 0.; + double tz = 0.; +#endif + for (int tid = 0; tid < num_threads; ++tid) { + fx += local_force(i, tid, 0); + fy += local_force(i, tid, 1); + fz += local_force(i, tid, 2); #ifdef ROTATION - tx += local_torque(i, tid, 0); - ty += local_torque(i, tid, 1); - tz += local_torque(i, tid, 2); -#endif - } - // auto &p = unique_particles.at(i); - // p->force() += Utils::Vector3d{fx, fy, fz}; - unique_particles.at(i)->force() += - Utils::Vector3d{fx, fy, fz}; + tx += local_torque(i, tid, 0); + ty += local_torque(i, tid, 1); + tz += local_torque(i, tid, 2); +#endif + } + // auto &p = unique_particles.at(i); + // p->force() += Utils::Vector3d{fx, fy, fz}; + unique_particles.at(i)->force() += + Utils::Vector3d{fx, fy, fz}; #ifdef ROTATION - // p->torque() += Utils::Vector3d{tx, ty, tz}; - unique_particles.at(i)->torque() += - Utils::Vector3d{tx, ty, tz}; + // p->torque() += Utils::Vector3d{tx, ty, tz}; + unique_particles.at(i)->torque() += + Utils::Vector3d{tx, ty, tz}; #endif - }); + }); Kokkos::fence(); #ifdef NPT @@ -329,8 +329,9 @@ void System::System::calculate_forces() { CALI_MARK_BEGIN("Cabana - Collision Detection"); #endif #ifdef COLLISION_DETECTION - auto collision_kernel = [&collision_detection = *collision_detection] - (Particle const &p1, Particle const &p2, Distance const &d) { + auto collision_kernel = [&collision_detection = *collision_detection]( + Particle const &p1, Particle const &p2, + Distance const &d) { if (not collision_detection.is_off()) { collision_detection.detect_collision(p1, p2, d.dist2); } diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 1cde4c9b968..5b7ffbd850b 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -128,7 +128,7 @@ __attribute__((always_inline)) inline void update_cabana_state( // Number of threads int num_threads = execution_space().concurrency(); - //int number_of_unique_particles = 0; + // int number_of_unique_particles = 0; bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); @@ -138,13 +138,14 @@ __attribute__((always_inline)) inline void update_cabana_state( // If we have to rebuild, we need to count the particles // cell_structure.set_index_map(); // parallelized index_map cell_structure.set_index_map(particles, ghost_particles); - + // Create essential variable for MD cell_structure.rebuild_local_properties( cell_structure.get_unique_particles().size(), num_threads, pair_cutoff); } else { // If we do not rebuild we can use the saved map - // number_of_unique_particles = cell_structure.get_unique_particles().size(); + // number_of_unique_particles = + // cell_structure.get_unique_particles().size(); cell_structure.reset_local_properties(); } auto const unique_particles = cell_structure.get_unique_particles(); @@ -200,11 +201,12 @@ __attribute__((always_inline)) inline void update_cabana_state( template -void cabana_short_range( - BondKernel const &bond_kernel, PairKernel const &forces_kernel, - CellStructure &cell_structure, double pair_cutoff, double bond_cutoff, - ParticleRange const &particles, ParticleRange const &ghost_particles, - VerletCriterion const &verlet_criterion = {}) { +void cabana_short_range(BondKernel const &bond_kernel, + PairKernel const &forces_kernel, + CellStructure &cell_structure, double pair_cutoff, + double bond_cutoff, ParticleRange const &particles, + ParticleRange const &ghost_particles, + VerletCriterion const &verlet_criterion = {}) { #ifdef CALIPER CALI_CXX_MARK_FUNCTION; #endif @@ -228,7 +230,8 @@ void cabana_short_range( #endif auto cabana_verlet_list = cell_structure.get_cabana_verlet_list(); // cabana_verlet_list.get_variance_max_counts(); - Kokkos::RangePolicy policy(0, cell_structure.get_unique_particles().size()); + Kokkos::RangePolicy policy( + 0, cell_structure.get_unique_particles().size()); Cabana::neighbor_parallel_for(policy, forces_kernel, cabana_verlet_list, Cabana::FirstNeighborsTag(), // Cabana::TeamOpTag()); From 95ace64a3c4b85af345caa33626ffdae680d5429 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 5 Aug 2025 18:29:09 +0200 Subject: [PATCH 77/94] Fixed missing hpp file --- src/core/cell_system/particle_enumeration.hpp | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/core/cell_system/particle_enumeration.hpp diff --git a/src/core/cell_system/particle_enumeration.hpp b/src/core/cell_system/particle_enumeration.hpp new file mode 100644 index 00000000000..89adf69c0f4 --- /dev/null +++ b/src/core/cell_system/particle_enumeration.hpp @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2010-2024 The ESPResSo project + * + * This file is part of ESPResSo. + * + * ESPResSo is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * ESPResSo is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "Cell.hpp" +#include "config/config.hpp" + +#include +#include + +#ifdef SHARED_MEMORY_PARALLELISM +#include +#endif + +// Forward declaration +struct CellStructure; + +/** + * @brief Run a kernel on all local particles with enumeration. + * The kernel is called with (index, particle) and is assumed to be thread-safe. + * + * @tparam Kernel Callable with signature void(int, Particle&) + * @param cs The CellStructure containing the particles + * @param kernel The kernel to apply to each particle with its index + */ +template +void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel); + +// Include the implementation +#include "CellStructure.hpp" + +template +void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel) { +#ifdef SHARED_MEMORY_PARALLELISM + if (cs.use_parallel_for_each_local_particle()) { + auto const local_cells = cs.decomposition().local_cells(); + + // Step 1: Calculate cell offsets + std::vector cell_offsets(local_cells.size() + 1, 0); + + // Calculate cumulative sum of particles per cell + for (size_t i = 0; i < local_cells.size(); ++i) { + cell_offsets[i + 1] = + cell_offsets[i] + local_cells[i]->particles().size(); + } + + // Step 2: Parallel loop over cells + Kokkos::parallel_for( + "enumerate_local_particles", local_cells.size(), [&](auto cell_idx) { + auto const base_offset = cell_offsets[cell_idx]; + auto &cell_particles = local_cells[cell_idx]->particles(); + + // Loop over particles in this cell + for (size_t part_idx = 0; part_idx < cell_particles.size(); + ++part_idx) { + int global_index = base_offset + part_idx; + kernel(global_index, *(cell_particles.begin() + part_idx)); + } + }); + return; + } +#endif + // Sequential fallback + int index = 0; + for (auto &p : cs.local_particles()) { + kernel(index++, p); + } +} \ No newline at end of file From 30704ab6f861e315288f08e9865d94b706251aea Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 5 Aug 2025 20:30:25 +0200 Subject: [PATCH 78/94] Fixed bugs --- src/core/cell_system/CellStructure.cpp | 17 +++++++++-------- src/core/cell_system/CellStructure.hpp | 6 +++--- src/core/cell_system/particle_enumeration.hpp | 4 ++-- src/core/forces_cabana.hpp | 12 ++++++------ src/core/short_range_cabana.hpp | 12 ++++-------- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 2514105d679..373a86b49c2 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -177,17 +177,18 @@ void CellStructure::reset_local_properties() { } void CellStructure::set_index_map() { -#ifdef CALIPER - CALI_CXX_MARK_FUNCTION; -#endif - m_unique_particles.clear(); - m_unique_particles.resize(count_local_particles()); +//#ifdef CALIPER +// CALI_CXX_MARK_FUNCTION; +//#endif + auto &unique_particles = m_unique_particles; + unique_particles.clear(); + unique_particles.resize(count_local_particles()); std::unordered_set registered_index{}; using execution_space = Kokkos::DefaultExecutionSpace; int n_threads = execution_space().concurrency(); std::vector max_ids(n_threads); - enumerate_local_particles(*this, [&](int index, Particle &p) { - m_unique_particles[index] = &p; + enumerate_local_particles(*this, [&unique_particles, &max_ids](int index, Particle &p) { + unique_particles[index] = &p; const int thread_num = omp_get_thread_num(); max_ids[thread_num] = std::max(p.id(), max_ids[thread_num]); }); @@ -204,7 +205,7 @@ void CellStructure::set_index_map() { continue; } registered_index.insert(p.id()); - m_unique_particles.emplace_back(&p); + unique_particles.emplace_back(&p); max_id = std::max(p.id(), max_id); } registered_index.clear(); diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 39cca8122be..91982b11225 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -85,7 +85,7 @@ const int vector_length = 1; using ForceType = Kokkos::View; using VirialType = Kokkos::View; -using data_types = Cabana::MemberTypes; +using data_types = Cabana::MemberTypes; using memory_space = Kokkos::HostSpace; using AoSoAType = Cabana::AoSoA>; @@ -343,8 +343,8 @@ struct CellStructure : public System::Leaf { return Cells::particles(decomposition().ghost_cells()); } - int count_local_particles() const { - int count = 0; + std::size_t count_local_particles() const { + std::size_t count = 0; for (auto const &cell : m_decomposition->local_cells()) { count += cell->particles().size(); } diff --git a/src/core/cell_system/particle_enumeration.hpp b/src/core/cell_system/particle_enumeration.hpp index 89adf69c0f4..f5f59d3de92 100644 --- a/src/core/cell_system/particle_enumeration.hpp +++ b/src/core/cell_system/particle_enumeration.hpp @@ -46,7 +46,7 @@ void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel); // Include the implementation #include "CellStructure.hpp" -template +template inline void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel) { #ifdef SHARED_MEMORY_PARALLELISM if (cs.use_parallel_for_each_local_particle()) { @@ -82,4 +82,4 @@ void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel) { for (auto &p : cs.local_particles()) { kernel(index++, p); } -} \ No newline at end of file +} diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index fdc51026ffd..28b312296c2 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -44,12 +44,12 @@ struct ForcesKernel { #if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) std::vector &unique_particles; #endif - ForceType local_force; + ForceType &local_force; #ifdef ROTATION - ForceType local_torque; + ForceType &local_torque; #endif #ifdef NPT - VirialType local_virial; + VirialType &local_virial; #endif const AoSoA_pack &aosoa; @@ -67,12 +67,12 @@ struct ForcesKernel { #if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) std::vector &unique_particles_, #endif - ForceType local_force_, + ForceType &local_force_, #ifdef ROTATION - ForceType local_torque_, + ForceType &local_torque_, #endif #ifdef NPT - VirialType local_virial_, + VirialType &local_virial_, #endif const AoSoA_pack &aosoa_) : bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 5b7ffbd850b..726b08273b2 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -128,30 +128,26 @@ __attribute__((always_inline)) inline void update_cabana_state( // Number of threads int num_threads = execution_space().concurrency(); - // int number_of_unique_particles = 0; - bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); // std::cout << "rebuild:" << rebuild << std::endl; if (rebuild) { // If we have to rebuild, we need to count the particles - // cell_structure.set_index_map(); // parallelized index_map - cell_structure.set_index_map(particles, ghost_particles); + cell_structure.set_index_map(); // parallelized index_map + // cell_structure.set_index_map(particles, ghost_particles); // Create essential variable for MD cell_structure.rebuild_local_properties( cell_structure.get_unique_particles().size(), num_threads, pair_cutoff); } else { // If we do not rebuild we can use the saved map - // number_of_unique_particles = - // cell_structure.get_unique_particles().size(); cell_structure.reset_local_properties(); } auto const unique_particles = cell_structure.get_unique_particles(); auto aosoa = cell_structure.get_aosoa_data(); - // int max_id = cell_structure.get_cached_max_local_particle_id(); - int max_id = cell_structure.get_max_id(); + int max_id = cell_structure.get_cached_max_local_particle_id(); + // int max_id = cell_structure.get_max_id(); #ifdef CALIPER CALI_MARK_END("Cabana - Index map"); From f94cd6c44136e56bb1230d07724dfcd67de3cde0 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Tue, 5 Aug 2025 20:31:24 +0200 Subject: [PATCH 79/94] Formatting --- src/core/cell_system/CellStructure.cpp | 17 +++++++++-------- src/core/cell_system/CellStructure.hpp | 2 +- src/core/cell_system/particle_enumeration.hpp | 5 +++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 373a86b49c2..54f33dbc1d8 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -177,9 +177,9 @@ void CellStructure::reset_local_properties() { } void CellStructure::set_index_map() { -//#ifdef CALIPER -// CALI_CXX_MARK_FUNCTION; -//#endif + // #ifdef CALIPER + // CALI_CXX_MARK_FUNCTION; + // #endif auto &unique_particles = m_unique_particles; unique_particles.clear(); unique_particles.resize(count_local_particles()); @@ -187,11 +187,12 @@ void CellStructure::set_index_map() { using execution_space = Kokkos::DefaultExecutionSpace; int n_threads = execution_space().concurrency(); std::vector max_ids(n_threads); - enumerate_local_particles(*this, [&unique_particles, &max_ids](int index, Particle &p) { - unique_particles[index] = &p; - const int thread_num = omp_get_thread_num(); - max_ids[thread_num] = std::max(p.id(), max_ids[thread_num]); - }); + enumerate_local_particles( + *this, [&unique_particles, &max_ids](int index, Particle &p) { + unique_particles[index] = &p; + const int thread_num = omp_get_thread_num(); + max_ids[thread_num] = std::max(p.id(), max_ids[thread_num]); + }); int max_id = *(std::max_element(max_ids.begin(), max_ids.end())); for (auto &p : ghost_particles()) { const Particle *local_particle = get_local_particle(p.id()); diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 91982b11225..b02d577f01d 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -344,7 +344,7 @@ struct CellStructure : public System::Leaf { } std::size_t count_local_particles() const { - std::size_t count = 0; + std::size_t count = 0; for (auto const &cell : m_decomposition->local_cells()) { count += cell->particles().size(); } diff --git a/src/core/cell_system/particle_enumeration.hpp b/src/core/cell_system/particle_enumeration.hpp index f5f59d3de92..0b0ab14c8ee 100644 --- a/src/core/cell_system/particle_enumeration.hpp +++ b/src/core/cell_system/particle_enumeration.hpp @@ -46,8 +46,9 @@ void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel); // Include the implementation #include "CellStructure.hpp" -template inline -void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel) { +template +inline void enumerate_local_particles(CellStructure const &cs, + Kernel &&kernel) { #ifdef SHARED_MEMORY_PARALLELISM if (cs.use_parallel_for_each_local_particle()) { auto const local_cells = cs.decomposition().local_cells(); From 2c20418df7198b230e60c36679ebcbb9ee8b6f42 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Wed, 6 Aug 2025 13:10:58 +0200 Subject: [PATCH 80/94] Deleted unnecessary comments --- src/core/cell_system/CellStructure.cpp | 28 -------------------------- src/core/cell_system/CellStructure.hpp | 10 --------- src/core/custom_verlet_list.hpp | 6 +++--- src/core/forces_inline.hpp | 9 --------- src/core/short_range_cabana.hpp | 10 ++------- 5 files changed, 5 insertions(+), 58 deletions(-) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 54f33dbc1d8..4aef8da0245 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -58,7 +58,6 @@ #ifdef SHARED_MEMORY_PARALLELISM #include "aosoa_pack.hpp" -#include "cabana_data.hpp" #include "custom_verlet_list.hpp" #include #include @@ -71,9 +70,6 @@ #ifdef SHARED_MEMORY_PARALLELISM CellStructure::~CellStructure() { - if (m_cabana_data) { - m_cabana_data.reset(); - } if (m_local_force) { m_local_force.reset(); } @@ -98,18 +94,8 @@ CellStructure::~CellStructure() { } } -void CellStructure::set_cabana_data(std::unique_ptr data) { - m_cabana_data = std::move(data); - m_rebuild_cabana_verlet_list = false; -} - -CabanaData &CellStructure::get_cabana_data() { return *m_cabana_data; } - void CellStructure::reset_cabana_data() { m_rebuild_verlet_list = true; - if (m_cabana_data) { - m_cabana_data.reset(); - } if (m_local_force) { m_local_force.reset(); } @@ -160,26 +146,12 @@ void CellStructure::reset_local_properties() { #ifdef ROTATION Kokkos::deep_copy(get_local_torque(), 0); #endif - /* - Kokkos::parallel_for(get_local_force().extent(0), [&](int i) { - for (int j = 0; j < get_local_force().extent(1); j++) { - for (int k : {0, 1, 2}) { - get_local_force()(i, j, k) = 0.; -#ifdef ROTATION - get_local_torque()(i, j, k) = 0.; -#endif - } - } - });*/ #ifdef NPT Kokkos::deep_copy(get_local_virial(), 0); #endif } void CellStructure::set_index_map() { - // #ifdef CALIPER - // CALI_CXX_MARK_FUNCTION; - // #endif auto &unique_particles = m_unique_particles; unique_particles.clear(); unique_particles.resize(count_local_particles()); diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index b02d577f01d..4ef6ca8b66e 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -78,7 +78,6 @@ template struct MemberTypes; template class AoSoA; } // namespace Cabana -class CabanaData; struct AoSoA_pack; // To construct AoSoA, vector_length is defined HERE. const int vector_length = 1; @@ -719,16 +718,12 @@ struct CellStructure : public System::Leaf { #ifdef SHARED_MEMORY_PARALLELISM private: - std::unique_ptr m_cabana_data; - // bool steepest_descent_flag = true; int max_prefactor = 8; int max_counts = -1; int m_max_id = 0; inline int estimate_max_counts(const double pair_cutoff, const int number_of_unique_particles) { - // std::cout << "estimate_max_counts:" << pair_cutoff << " " - // << max_prefactor << std::endl; int max_counts; if (not std::isinf(pair_cutoff)) { max_counts = static_cast( @@ -747,8 +742,6 @@ struct CellStructure : public System::Leaf { } public: - void set_cabana_data(std::unique_ptr data); - CabanaData &get_cabana_data(); void reset_cabana_data(); virtual ~CellStructure(); @@ -761,9 +754,6 @@ struct CellStructure : public System::Leaf { m_rebuild_cabana_verlet_list = false; } - // void set_steepest_descent_flag(bool flag) { steepest_descent_flag = flag; } - // bool get_steepest_descent_flag() { return steepest_descent_flag; } - void set_max_prefactor(int value) { max_prefactor = value; } void set_max_counts(int value) { max_counts = value; } diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 6938feb1457..71a1fc5bb44 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -59,7 +59,7 @@ class CustomVerletList // Method to add a neighbor KOKKOS_INLINE_FUNCTION - void addNeighborAtomic(int pid, int nid) { + void addNeighborAtomicLB(int pid, int nid) { std::size_t count = counts(pid); std::size_t count_n = counts(nid); @@ -80,7 +80,7 @@ class CustomVerletList // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION - void addNeighborNonAtomic(int pid, int nid) { + void addNeighbor(int pid, int nid) { std::size_t count = counts(pid); #ifndef NDEBUG @@ -95,7 +95,7 @@ class CustomVerletList // Non atomic and load balancing method to add a neighbor KOKKOS_INLINE_FUNCTION - void addNeighborLoadBalancing(int pid, int nid) { + void addNeighborLB(int pid, int nid) { std::size_t count = counts(pid); std::size_t count_n = counts(nid); diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index f1342d7c423..3687dfe6df9 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -183,8 +183,6 @@ inline void add_non_bonded_pair_without_p( IA_parameters const &ia_params, [[maybe_unused]] bool do_nonbonded, Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel) { - // ParticleForce pf{}; - /***********************************************/ /* non-bonded pair potentials */ /***********************************************/ @@ -205,12 +203,10 @@ inline void add_non_bonded_pair_without_p( #ifdef ELECTROSTATICS // real-space electrostatic charge-charge interaction - // auto const q1q2 = p1.q() * p2.q(); if (q1q2 != 0. and coulomb_kernel != nullptr) { pf.f += (*coulomb_kernel)(q1q2, d, dist); } #endif // ELECTROSTATICS - // return pf; } /** @@ -230,9 +226,6 @@ inline void add_non_bonded_pair_force_with_p( Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel) { - // ParticleForce pf{}; - // Utils::Vector3d virial{}; - /***********************************************/ /* non-bonded pair potentials */ /***********************************************/ @@ -271,7 +264,6 @@ inline void add_non_bonded_pair_force_with_p( #ifdef ELECTROSTATICS // real-space electrostatic charge-charge interaction - // auto const q1q2 = p1.q() * p2.q(); if (q1q2 != 0. and coulomb_kernel != nullptr) { // pf.f += (*coulomb_kernel)(q1q2, d, dist); #ifdef NPT @@ -313,7 +305,6 @@ inline void add_non_bonded_pair_force_with_p( pf += (*dipoles_kernel)(p1, p2, d, dist, dist2); } #endif - // return std::pair{pf, virial}; } /** Calculate non-bonded forces between a pair of particles and update their diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 726b08273b2..755a61393fb 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -74,7 +74,7 @@ __attribute__((always_inline)) inline void construct_verlet_list( if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { int jj = id_to_index((*jt).id()); if (jj >= 0) { - verlet_list.addNeighborLoadBalancing(ii, jj); + verlet_list.addNeighborLB(ii, jj); } } } @@ -99,8 +99,7 @@ __attribute__((always_inline)) inline void construct_verlet_list( if (verlet_criterion(p1, p2, distance_function(p1, p2))) { int jj = id_to_index(p2.id()); if (jj >= 0) { - // verlet_list.addNeighborAtomic(ii, jj); - verlet_list.addNeighborNonAtomic(ii, jj); + verlet_list.addNeighbor(ii, jj); } } } @@ -113,8 +112,6 @@ __attribute__((always_inline)) inline void construct_verlet_list( Kokkos::parallel_for("intra", cells.size(), inter_kernel); Kokkos::fence(); - - // verlet_list.sortNeighbors(); } template @@ -130,12 +127,10 @@ __attribute__((always_inline)) inline void update_cabana_state( bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); - // std::cout << "rebuild:" << rebuild << std::endl; if (rebuild) { // If we have to rebuild, we need to count the particles cell_structure.set_index_map(); // parallelized index_map - // cell_structure.set_index_map(particles, ghost_particles); // Create essential variable for MD cell_structure.rebuild_local_properties( @@ -147,7 +142,6 @@ __attribute__((always_inline)) inline void update_cabana_state( auto const unique_particles = cell_structure.get_unique_particles(); auto aosoa = cell_structure.get_aosoa_data(); int max_id = cell_structure.get_cached_max_local_particle_id(); - // int max_id = cell_structure.get_max_id(); #ifdef CALIPER CALI_MARK_END("Cabana - Index map"); From 9d4310ca3a5f1541a0c9b1f0abbc719659ef5a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Wed, 6 Aug 2025 15:21:07 +0200 Subject: [PATCH 81/94] Refactor --- src/core/cell_system/CellStructure.cpp | 37 ++++++-------------------- src/core/cell_system/CellStructure.hpp | 8 +++--- src/core/communication.cpp | 11 +++++--- src/core/communication.hpp | 10 ++++--- src/core/forces_cabana.hpp | 10 ++++++- src/core/forces_inline.hpp | 1 - src/core/short_range_cabana.hpp | 7 +++-- src/core/system/System.cpp | 9 +++---- src/core/system/System.hpp | 2 -- 9 files changed, 42 insertions(+), 53 deletions(-) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 4aef8da0245..37409e42b54 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -94,35 +94,14 @@ CellStructure::~CellStructure() { } } -void CellStructure::reset_cabana_data() { - m_rebuild_verlet_list = true; - if (m_local_force) { - m_local_force.reset(); - } -#ifdef ROTATION - if (m_local_torque) { - m_local_torque.reset(); - } -#endif -#ifdef NPT - if (m_local_virial) { - m_local_virial.reset(); - } -#endif - if (m_aosoa) { - m_aosoa.reset(); - } - if (m_particle_storage) { - m_particle_storage.reset(); - } - if (m_cabana_verlet_list) { - m_cabana_verlet_list.reset(); - } +void CellStructure::set_kokkos_handle(std::shared_ptr handle) { + m_kokkos_handle = std::move(handle); } -void CellStructure::rebuild_local_properties(const std::size_t num_part, - const std::size_t num_threads, - const double pair_cutoff) { +void CellStructure::rebuild_local_properties(std::size_t const num_threads, + double const pair_cutoff) { + assert(m_kokkos_handle); + auto const num_part = get_unique_particles().size(); m_local_force = std::make_unique("local_force", num_part, num_threads); #ifdef ROTATION @@ -133,7 +112,7 @@ void CellStructure::rebuild_local_properties(const std::size_t num_part, m_local_virial = std::make_unique("local_virial", num_threads); #endif m_particle_storage = std::make_unique("particles", num_part); - (*m_particle_storage).resize(num_part); + m_particle_storage->resize(num_part); // particle properties are defined in aosoa_pack.hpp m_aosoa = std::make_unique(*m_particle_storage); @@ -167,7 +146,7 @@ void CellStructure::set_index_map() { }); int max_id = *(std::max_element(max_ids.begin(), max_ids.end())); for (auto &p : ghost_particles()) { - const Particle *local_particle = get_local_particle(p.id()); + auto const *local_particle = get_local_particle(p.id()); if (not local_particle) { continue; } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 4ef6ca8b66e..e6c3ad54e71 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -79,6 +79,7 @@ template class AoSoA; } // namespace Cabana struct AoSoA_pack; +struct KokkosHandle; // To construct AoSoA, vector_length is defined HERE. const int vector_length = 1; @@ -721,6 +722,7 @@ struct CellStructure : public System::Leaf { int max_prefactor = 8; int max_counts = -1; int m_max_id = 0; + std::shared_ptr m_kokkos_handle; inline int estimate_max_counts(const double pair_cutoff, const int number_of_unique_particles) { @@ -742,8 +744,6 @@ struct CellStructure : public System::Leaf { } public: - void reset_cabana_data(); - virtual ~CellStructure(); bool get_rebuild_verlet_list() const { return m_rebuild_verlet_list; } @@ -761,8 +761,8 @@ struct CellStructure : public System::Leaf { int get_max_id() { return m_max_id; } - void rebuild_local_properties(std::size_t num_part, std::size_t num_threads, - double pair_cutoff); + void set_kokkos_handle(std::shared_ptr handle); + void rebuild_local_properties(std::size_t num_threads, double pair_cutoff); void reset_local_properties(); ForceType &get_local_force() { return *m_local_force; } diff --git a/src/core/communication.cpp b/src/core/communication.cpp index e7beaf0d755..027c5a52a44 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -55,7 +55,13 @@ #include #include +struct KokkosHandle { + KokkosHandle() { Kokkos::initialize(); } + ~KokkosHandle() { Kokkos::finalize(); } +}; + boost::mpi::communicator comm_cart; +std::shared_ptr kokkos_handle; Communicator communicator{}; namespace Communication { @@ -108,8 +114,7 @@ void init(std::shared_ptr mpi_env) { #endif #ifdef SHARED_MEMORY_PARALLELISM - Kokkos::initialize(); - // Kokkos::print_configuration(std::cout); + kokkos_handle = std::make_shared(); #endif } @@ -118,7 +123,7 @@ void deinit() { Communication::m_callbacks.reset(); #ifdef SHARED_MEMORY_PARALLELISM - Kokkos::finalize(); + kokkos_handle.reset(); #endif } } // namespace Communication diff --git a/src/core/communication.hpp b/src/core/communication.hpp index a18046777cc..568d935b282 100644 --- a/src/core/communication.hpp +++ b/src/core/communication.hpp @@ -18,8 +18,9 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -#ifndef CORE_COMMUNICATION_HPP -#define CORE_COMMUNICATION_HPP + +#pragma once + /** \file * This file contains the asynchronous MPI communication. * @@ -58,6 +59,10 @@ extern int this_node; /** The communicator */ extern boost::mpi::communicator comm_cart; +#ifdef SHARED_MEMORY_PARALLELISM +struct KokkosHandle; +extern std::shared_ptr kokkos_handle; +#endif struct Communicator { boost::mpi::communicator &comm; @@ -117,4 +122,3 @@ struct MpiContainerUnitTest { } ~MpiContainerUnitTest() { Communication::deinit(); } }; -#endif diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index 28b312296c2..3b06f9b0917 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -19,6 +19,8 @@ #pragma once +#include "config/config.hpp" + #ifdef CALIPER #include #endif @@ -30,6 +32,12 @@ #include +#if defined(__GNUG__) or defined(__clang__) +#define ESPRESSO_ATTR_ALWAYS_INLINE [[gnu::always_inline]] +#else +#define ESPRESSO_ATTR_ALWAYS_INLINE +#endif + struct ForcesKernel { [[maybe_unused]] const BondedInteractionsMap &bonded_ias; const InteractionsNonBonded &nonbonded_ias; @@ -95,7 +103,7 @@ struct ForcesKernel { aosoa(aosoa_) { } - __attribute__((always_inline)) KOKKOS_INLINE_FUNCTION void + ESPRESSO_ATTR_ALWAYS_INLINE KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { auto thread_id = omp_get_thread_num(); diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 3687dfe6df9..b36c0d1c14e 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -76,7 +76,6 @@ #include -#include #include #include #include diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 755a61393fb..fe161954b0d 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -48,7 +48,7 @@ inline void write_particle(Particle const &p, int const &id, } template -__attribute__((always_inline)) inline void construct_verlet_list( +ESPRESSO_ATTR_ALWAYS_INLINE inline void construct_verlet_list( CellStructure &cell_structure, VerletCriterion const &verlet_criterion, Kokkos::View const &id_to_index, const int max_id) { auto const &cells = @@ -115,7 +115,7 @@ __attribute__((always_inline)) inline void construct_verlet_list( } template -__attribute__((always_inline)) inline void update_cabana_state( +ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( CellStructure &cell_structure, ParticleRange const &particles, ParticleRange const &ghost_particles, VerletCriterion const &verlet_criterion, double const pair_cutoff) { @@ -133,8 +133,7 @@ __attribute__((always_inline)) inline void update_cabana_state( cell_structure.set_index_map(); // parallelized index_map // Create essential variable for MD - cell_structure.rebuild_local_properties( - cell_structure.get_unique_particles().size(), num_threads, pair_cutoff); + cell_structure.rebuild_local_properties(num_threads, pair_cutoff); } else { // If we do not rebuild we can use the saved map cell_structure.reset_local_properties(); diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index cb5d7851da1..c96819568a4 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -66,6 +66,9 @@ System::System(Private) { box_geo = std::make_shared(); local_geo = std::make_shared(); cell_structure = std::make_shared(*box_geo); +#ifdef SHARED_MEMORY_PARALLELISM + cell_structure->set_kokkos_handle(::kokkos_handle); +#endif propagation = std::make_shared(); bonded_ias = std::make_shared(); thermostat = std::make_shared(); @@ -94,12 +97,6 @@ System::System(Private) { min_global_cut = INACTIVE_CUTOFF; } -System::~System() { -#ifdef SHARED_MEMORY_PARALLELISM - cell_structure->reset_cabana_data(); -#endif -} - void System::initialize() { auto handle = shared_from_this(); cell_structure->bind_system(handle); diff --git a/src/core/system/System.hpp b/src/core/system/System.hpp index 71545e0b4e7..ceb1103f33c 100644 --- a/src/core/system/System.hpp +++ b/src/core/system/System.hpp @@ -86,8 +86,6 @@ class System : public std::enable_shared_from_this { static std::shared_ptr create(); - virtual ~System(); - #ifdef CUDA GpuParticleData gpu; #endif From 28a9a735ed6dfca39b9681f372560b7aa3a48853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Wed, 6 Aug 2025 16:48:41 +0200 Subject: [PATCH 82/94] Refactor --- src/core/BoxGeometry.hpp | 6 +- src/core/aosoa_pack.hpp | 3 +- src/core/cabana_data.hpp | 59 ------------------- src/core/cell_system/CellStructure.cpp | 41 +++++++++---- src/core/cell_system/CellStructure.hpp | 28 +-------- src/core/cell_system/particle_enumeration.hpp | 22 +++---- src/core/communication.cpp | 6 +- src/core/custom_verlet_list.hpp | 26 ++++---- src/core/forces.cpp | 12 ---- src/core/forces_cabana.hpp | 11 ++-- src/core/forces_inline.hpp | 2 - src/core/short_range_cabana.hpp | 45 ++++---------- src/core/system/System.cpp | 3 + testsuite/python/caliper.py | 1 - testsuite/python/integrator_npt_stats.py | 4 +- testsuite/python/scafacos_interface.py | 6 +- testsuite/python/unittest_decorators.py | 7 --- 17 files changed, 86 insertions(+), 196 deletions(-) delete mode 100644 src/core/cabana_data.hpp diff --git a/src/core/BoxGeometry.hpp b/src/core/BoxGeometry.hpp index 9f90836dd03..aad3fcedc67 100644 --- a/src/core/BoxGeometry.hpp +++ b/src/core/BoxGeometry.hpp @@ -240,9 +240,9 @@ class BoxGeometry { * periodic images, i.e. a - b. */ template - Utils::Vector get_mi_vector(const T &a0, const T &a1, const T &a2, - const T &b0, const T &b1, - const T &b2) const { + Utils::Vector get_mi_vector(T const &a0, T const &a1, T const &a2, + T const &b0, T const &b1, + T const &b2) const { if (type() == BoxType::LEES_EDWARDS) { auto const shear_plane_normal = lees_edwards_bc().shear_plane_normal; auto a_tmp = Utils::Vector{a0, a1, a2}; diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index a6611334ae3..e3ba514d9e3 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2025 The ESPResSo project + * Copyright (C) 2025 The ESPResSo project * * This file is part of ESPResSo. * @@ -38,4 +38,5 @@ struct AoSoA_pack { : position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)) {} }; + #endif diff --git a/src/core/cabana_data.hpp b/src/core/cabana_data.hpp deleted file mode 100644 index baa5853825f..00000000000 --- a/src/core/cabana_data.hpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2010-2025 The ESPResSo project - * - * This file is part of ESPResSo. - * - * ESPResSo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * ESPResSo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#pragma once - -#ifdef SHARED_MEMORY_PARALLELISM - -#include "custom_verlet_list.hpp" -#include -#include - -using memory_space = Kokkos::SharedSpace; -using execution_space = Kokkos::DefaultExecutionSpace; - -using ListAlgorithm = Cabana::HalfNeighborTag; -using ListType = Cabana::CustomVerletList; - -class CabanaData { -private: - ListType verlet_list; - std::vector unique_particles; - int max_id; - -public: - // CabanaData() = default; - CabanaData(ListType &verlet_list, std::vector &unique_particles) - : verlet_list(verlet_list), unique_particles(unique_particles) {} - CabanaData(ListType &verlet_list, std::vector &unique_particles, - int max_id) - : verlet_list(verlet_list), unique_particles(unique_particles), - max_id(max_id) {} - - ListType get_verlet_list() const { return verlet_list; } - int get_index() const { return unique_particles.size(); } - int get_max_id() const { return max_id; } - std::vector get_unique_particles() const { - return unique_particles; - } - - ~CabanaData() {}; -}; -#endif diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 29fe91cd0e7..939150f61ee 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -38,6 +38,7 @@ #include #include +#include #include #include @@ -63,13 +64,9 @@ #include #include #endif -#ifdef CALIPER -#include "caliper/cali.h" -#endif - -#ifdef SHARED_MEMORY_PARALLELISM CellStructure::~CellStructure() { +#ifdef SHARED_MEMORY_PARALLELISM if (m_local_force) { m_local_force.reset(); } @@ -92,12 +89,34 @@ CellStructure::~CellStructure() { if (m_cabana_verlet_list) { m_cabana_verlet_list.reset(); } +#endif } +#ifdef SHARED_MEMORY_PARALLELISM + void CellStructure::set_kokkos_handle(std::shared_ptr handle) { m_kokkos_handle = std::move(handle); } +static auto estimate_max_counts(int max_prefactor, double pair_cutoff, + std::size_t number_of_unique_particles) { + if (std::isinf(pair_cutoff)) { + return number_of_unique_particles; + } + auto const volume = Utils::int_pow<3>(pair_cutoff); + auto max_counts = static_cast( + std::ceil(static_cast(max_prefactor) * volume)); +#ifdef COLLISION_DETECTION + std::size_t constexpr threshold_num = 64; +#else + std::size_t constexpr threshold_num = 16; +#endif + if (max_counts < threshold_num) { + max_counts = std::min(threshold_num, number_of_unique_particles); + } + return max_counts; +} + void CellStructure::rebuild_local_properties(std::size_t const num_threads, double const pair_cutoff) { assert(m_kokkos_handle); @@ -116,8 +135,8 @@ void CellStructure::rebuild_local_properties(std::size_t const num_threads, // particle properties are defined in aosoa_pack.hpp m_aosoa = std::make_unique(*m_particle_storage); - int max_counts = estimate_max_counts(pair_cutoff, num_part); - m_cabana_verlet_list = std::make_unique(0, num_part, max_counts); + auto max_counts = estimate_max_counts(max_prefactor, pair_cutoff, num_part); + m_cabana_verlet_list = std::make_unique(0ul, num_part, max_counts); } void CellStructure::reset_local_properties() { @@ -139,7 +158,7 @@ void CellStructure::set_index_map() { int n_threads = execution_space().concurrency(); std::vector max_ids(n_threads); enumerate_local_particles( - *this, [&unique_particles, &max_ids](int index, Particle &p) { + *this, [&unique_particles, &max_ids](std::size_t index, Particle &p) { unique_particles[index] = &p; const int thread_num = omp_get_thread_num(); max_ids[thread_num] = std::max(p.id(), max_ids[thread_num]); @@ -163,7 +182,8 @@ void CellStructure::set_index_map() { registered_index.clear(); m_cached_max_local_particle_id = max_id; } -#endif + +#endif // SHARED_MEMORY_PARALLELISM CellStructure::CellStructure(BoxGeometry const &box) : m_decomposition{std::make_unique(box)} {} @@ -393,9 +413,6 @@ void CellStructure::set_verlet_skin(double value) { m_verlet_skin = value; m_verlet_skin_set = true; m_rebuild_cabana_verlet_list = true; -#ifdef SHARED_MEMORY_PARALLELISM - max_counts = -1; -#endif get_system().on_verlet_skin_change(); } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index e6c3ad54e71..85963c81c69 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -216,10 +216,11 @@ struct CellStructure : public System::Leaf { std::vector m_unique_particles; std::unique_ptr m_cabana_verlet_list; -#endif +#endif // SHARED_MEMORY_PARALLELISM public: CellStructure(BoxGeometry const &box); + virtual ~CellStructure(); bool use_verlet_list = true; @@ -720,32 +721,10 @@ struct CellStructure : public System::Leaf { #ifdef SHARED_MEMORY_PARALLELISM private: int max_prefactor = 8; - int max_counts = -1; int m_max_id = 0; std::shared_ptr m_kokkos_handle; - inline int estimate_max_counts(const double pair_cutoff, - const int number_of_unique_particles) { - int max_counts; - if (not std::isinf(pair_cutoff)) { - max_counts = static_cast( - std::ceil(max_prefactor * pair_cutoff * pair_cutoff * pair_cutoff)); - int threshold_num = 16; // 8; -#ifdef COLLISION_DETECTION - threshold_num = 64; -#endif - if (max_counts < threshold_num) { - max_counts = std::min(threshold_num, number_of_unique_particles); - } - } else { - max_counts = number_of_unique_particles; - } - return max_counts; - } - public: - virtual ~CellStructure(); - bool get_rebuild_verlet_list() const { return m_rebuild_verlet_list; } bool get_rebuild_cabana_verlet_list() const { return m_rebuild_cabana_verlet_list; @@ -756,9 +735,6 @@ struct CellStructure : public System::Leaf { void set_max_prefactor(int value) { max_prefactor = value; } - void set_max_counts(int value) { max_counts = value; } - int get_max_counts() const { return max_counts; } - int get_max_id() { return m_max_id; } void set_kokkos_handle(std::shared_ptr handle); diff --git a/src/core/cell_system/particle_enumeration.hpp b/src/core/cell_system/particle_enumeration.hpp index 0b0ab14c8ee..fa9e2adc575 100644 --- a/src/core/cell_system/particle_enumeration.hpp +++ b/src/core/cell_system/particle_enumeration.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2024 The ESPResSo project + * Copyright (C) 2025 The ESPResSo project * * This file is part of ESPResSo. * @@ -20,17 +20,16 @@ #pragma once #include "Cell.hpp" +#include "CellStructure.hpp" #include "config/config.hpp" -#include -#include - #ifdef SHARED_MEMORY_PARALLELISM #include #endif -// Forward declaration -struct CellStructure; +#include +#include +#include /** * @brief Run a kernel on all local particles with enumeration. @@ -43,9 +42,6 @@ struct CellStructure; template void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel); -// Include the implementation -#include "CellStructure.hpp" - template inline void enumerate_local_particles(CellStructure const &cs, Kernel &&kernel) { @@ -57,7 +53,7 @@ inline void enumerate_local_particles(CellStructure const &cs, std::vector cell_offsets(local_cells.size() + 1, 0); // Calculate cumulative sum of particles per cell - for (size_t i = 0; i < local_cells.size(); ++i) { + for (std::size_t i = 0; i < local_cells.size(); ++i) { cell_offsets[i + 1] = cell_offsets[i] + local_cells[i]->particles().size(); } @@ -69,7 +65,7 @@ inline void enumerate_local_particles(CellStructure const &cs, auto &cell_particles = local_cells[cell_idx]->particles(); // Loop over particles in this cell - for (size_t part_idx = 0; part_idx < cell_particles.size(); + for (std::size_t part_idx = 0; part_idx < cell_particles.size(); ++part_idx) { int global_index = base_offset + part_idx; kernel(global_index, *(cell_particles.begin() + part_idx)); @@ -77,9 +73,9 @@ inline void enumerate_local_particles(CellStructure const &cs, }); return; } -#endif +#endif // SHARED_MEMORY_PARALLELISM // Sequential fallback - int index = 0; + std::size_t index = 0; for (auto &p : cs.local_particles()) { kernel(index++, p); } diff --git a/src/core/communication.cpp b/src/core/communication.cpp index 027c5a52a44..8bfe287cbee 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -55,14 +55,18 @@ #include #include +#ifdef SHARED_MEMORY_PARALLELISM struct KokkosHandle { KokkosHandle() { Kokkos::initialize(); } ~KokkosHandle() { Kokkos::finalize(); } }; +#endif boost::mpi::communicator comm_cart; -std::shared_ptr kokkos_handle; Communicator communicator{}; +#ifdef SHARED_MEMORY_PARALLELISM +std::shared_ptr kokkos_handle{}; +#endif namespace Communication { static std::shared_ptr m_callbacks; diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 71a1fc5bb44..22544f35be7 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -21,7 +21,9 @@ #ifdef SHARED_MEMORY_PARALLELISM #include + #include +#include namespace Cabana { // ONLY FOR 2D LAYOUT, OTHERWISE NEIGHBOR LIST INTERFACE IMPLEMENTATION WILL @@ -37,8 +39,8 @@ class CustomVerletList CustomVerletList() : Base() {} // Custom constructor - CustomVerletList(const std::size_t begin, const std::size_t end, - const std::size_t max_neigh) { + CustomVerletList(std::size_t const begin, std::size_t const end, + std::size_t const max_neigh) { initializeData(end - begin, max_neigh); } virtual ~CustomVerletList() {}; @@ -49,8 +51,8 @@ class CustomVerletList // Method to initialize _data without filling neighbors KOKKOS_INLINE_FUNCTION - void initializeData(const std::size_t num_particles, - const std::size_t max_neigh) { + void initializeData(std::size_t const num_particles, + std::size_t const max_neigh) { counts = Kokkos::View("num_neighbors", num_particles); neighbors = Kokkos::View( Kokkos::ViewAllocateWithoutInitializing("neighbors"), num_particles, @@ -181,7 +183,7 @@ class NeighborList< //! Get the total number of neighbors across all particles. KOKKOS_INLINE_FUNCTION - static std::size_t totalNeighbor(const list_type &list) { + static std::size_t totalNeighbor(list_type const &list) { std::size_t total_n = 0; std::size_t num_p = list.counts.size(); for (std::size_t i = 0; i < num_p; ++i) @@ -191,28 +193,28 @@ class NeighborList< //! Get the maximum number of neighbors per particle. KOKKOS_INLINE_FUNCTION - static std::size_t maxNeighbor(const list_type &list) { + static std::size_t maxNeighbor(list_type const &list) { // Stored during neighbor search. return list.max_n; } //! Get the number of neighbors for a given particle index. KOKKOS_INLINE_FUNCTION - static std::size_t numNeighbor(const list_type &list, - const std::size_t particle_index) { + static std::size_t numNeighbor(list_type const &list, + std::size_t const particle_index) { return list.counts(particle_index); } //! Get the id for a neighbor for a given particle index and the index of //! the neighbor relative to the particle. KOKKOS_INLINE_FUNCTION - static std::size_t getNeighbor(const list_type &list, - const std::size_t particle_index, - const std::size_t count) { + static std::size_t getNeighbor(list_type const &list, + std::size_t const particle_index, + std::size_t const count) { return list.neighbors(particle_index, count); } }; } // namespace Cabana -#endif +#endif // SHARED_MEMORY_PARALLELISM diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 73285909e80..072bb02d8cf 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -265,9 +265,6 @@ void System::System::calculate_forces() { get_interaction_range(), bonded_ias->maximal_cutoff(), particles, cell_structure->ghost_particles(), verlet_criterion); -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - reduction Forces"); -#endif // Force and Torque reduction int num_threads = execution_space().concurrency(); Kokkos::RangePolicy policy(0, unique_particles.size()); @@ -319,13 +316,7 @@ void System::System::calculate_forces() { Utils::Vector3d virial_vec{vx, vy, vz}; npt_add_virial_force_contribution(virial_vec); #endif -#ifdef CALIPER - CALI_MARK_END("Cabana - reduction Forces"); -#endif -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Collision Detection"); -#endif #ifdef COLLISION_DETECTION auto collision_kernel = [&collision_detection = *collision_detection]( Particle const &p1, Particle const &p2, @@ -336,9 +327,6 @@ void System::System::calculate_forces() { }; cell_structure->non_bonded_loop(collision_kernel, verlet_criterion); #endif -#ifdef CALIPER - CALI_MARK_END("Cabana - Collision Detection"); -#endif #else // SHARED_MEMORY_PARALLELISM diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index 3b06f9b0917..40b1c18bb1b 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2025 The ESPResSo project + * Copyright (C) 2025 The ESPResSo project * * This file is part of ESPResSo. * @@ -21,10 +21,6 @@ #include "config/config.hpp" -#ifdef CALIPER -#include -#endif - #ifdef SHARED_MEMORY_PARALLELISM #include "aosoa_pack.hpp" @@ -32,6 +28,8 @@ #include +#include + #if defined(__GNUG__) or defined(__clang__) #define ESPRESSO_ATTR_ALWAYS_INLINE [[gnu::always_inline]] #else @@ -172,4 +170,5 @@ struct ForcesKernel { #endif } }; -#endif + +#endif // SHARED_MEMORY_PARALLELISM diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 0aab1c0139a..64ca1f98503 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -232,7 +232,6 @@ inline void add_non_bonded_pair_force_with_p( #ifdef EXCLUSIONS if (do_nonbonded) { #endif - // pf += calc_central_radial_force(ia_params, d, dist); #ifdef THOLE pf.f += thole_pair_force(p1, p2, ia_params, d, dist, bonded_ias, coulomb_kernel); @@ -263,7 +262,6 @@ inline void add_non_bonded_pair_force_with_p( #ifdef ELECTROSTATICS // real-space electrostatic charge-charge interaction if (q1q2 != 0. and coulomb_kernel != nullptr) { - // pf.f += (*coulomb_kernel)(q1q2, d, dist); #ifdef NPT #ifdef SHARED_MEMORY_PARALLELISM virial[0] += (*coulomb_u_kernel)(p1, p2, q1q2, d, dist); diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index fe161954b0d..58c2a848f94 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2025 The ESPResSo project + * Copyright (C) 2025 The ESPResSo project * * This file is part of ESPResSo. * @@ -21,27 +21,26 @@ #include "config/config.hpp" -#include "cell_system/CellStructure.hpp" - -#ifdef CALIPER -#include -#endif - #ifdef SHARED_MEMORY_PARALLELISM +#include "cell_system/CellStructure.hpp" + #include "aosoa_pack.hpp" #include "custom_verlet_list.hpp" #include "forces_cabana.hpp" + #include #include -#include + +#include +#include inline void write_particle(Particle const &p, int const &id, AoSoA_pack &aosoa) { aosoa.id(id) = p.id(); aosoa.charge(id) = p.q(); aosoa.type(id) = p.type(); - auto const pos = p.pos(); + auto const &pos = p.pos(); for (int d = 0; d < 3; ++d) { aosoa.position(id, d) = pos[d]; } @@ -61,7 +60,7 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void construct_verlet_list( &id_to_index, &verlet_list, max_id](const int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { - auto &p1 = *it; + auto const &p1 = *it; if (p1.id() > max_id) continue; int ii = id_to_index(p1.id()); @@ -119,9 +118,6 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( CellStructure &cell_structure, ParticleRange const &particles, ParticleRange const &ghost_particles, VerletCriterion const &verlet_criterion, double const pair_cutoff) { -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Index map"); -#endif // Number of threads int num_threads = execution_space().concurrency(); @@ -142,14 +138,8 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( auto aosoa = cell_structure.get_aosoa_data(); int max_id = cell_structure.get_cached_max_local_particle_id(); -#ifdef CALIPER - CALI_MARK_END("Cabana - Index map"); -#endif // Fill the essential variable for MD { -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Allocation"); -#endif // =================================================== // Fill particle storage // =================================================== @@ -165,9 +155,6 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( id_to_index(unique_particles.at(p_id)->id()) = p_id; }); Kokkos::fence(); -#ifdef CALIPER - CALI_MARK_END("Cabana - Allocation"); -#endif // =================================================== // Get Verlet Pairs and Fill Verlet list @@ -175,15 +162,9 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( // Rebuild verlet list if needed if (rebuild) { -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - Verlet List"); -#endif construct_verlet_list(cell_structure, verlet_criterion, id_to_index, max_id); cell_structure.mark_rebuild_cabana_verlet_list_as_UpToDate(); -#ifdef CALIPER - CALI_MARK_END("Cabana - Verlet List"); -#endif } } } @@ -214,9 +195,6 @@ void cabana_short_range(BondKernel const &bond_kernel, // Cabana short range loop if (pair_cutoff > 0.) { -#ifdef CALIPER - CALI_MARK_BEGIN("Cabana - calc Force"); -#endif auto cabana_verlet_list = cell_structure.get_cabana_verlet_list(); // cabana_verlet_list.get_variance_max_counts(); Kokkos::RangePolicy policy( @@ -226,10 +204,7 @@ void cabana_short_range(BondKernel const &bond_kernel, // Cabana::TeamOpTag()); Cabana::SerialOpTag()); Kokkos::fence(); -#ifdef CALIPER - CALI_MARK_END("Cabana - calc Force"); -#endif } } -#endif +#endif // SHARED_MEMORY_PARALLELISM diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index c96819568a4..65b2e792e9c 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -432,6 +432,9 @@ bool System::long_range_interactions_sanity_checks() const { } double System::get_interaction_range() const { + if (maximal_cutoff() > 1000000.) { + auto const max_cut = maximal_cutoff(); + } auto const max_cut = maximal_cutoff(); auto const verlet_skin = cell_structure->get_verlet_skin(); /* Consider skin only if there are actually interactions */ diff --git a/testsuite/python/caliper.py b/testsuite/python/caliper.py index 55fc55c48f2..265e2f741e7 100644 --- a/testsuite/python/caliper.py +++ b/testsuite/python/caliper.py @@ -49,7 +49,6 @@ @utx.skipIfMissingFeatures(["CALIPER"]) class Test(ut.TestCase): - @utx.skipIfExistingFeatures(["SHARED_MEMORY_PARALLELISM"]) @utx.skipIfMissingFeatures(["P3M", "WCA"]) def test_runtime_report(self): has_cuda = espressomd.has_features(["CUDA"]) diff --git a/testsuite/python/integrator_npt_stats.py b/testsuite/python/integrator_npt_stats.py index c25044e6d6d..ed352fe1a1c 100644 --- a/testsuite/python/integrator_npt_stats.py +++ b/testsuite/python/integrator_npt_stats.py @@ -113,8 +113,8 @@ def test_compressibility_and_pressure(self): self.assertAlmostEqual(avp, p_ext, delta=0.02) self.assertAlmostEqual(compressibility, 0.5, delta=0.05) np.testing.assert_allclose(avp_sim_vir, avp_inst_vir, atol=1e-10) - self.assertAlmostEqual(avpV_sim, 100., delta=1.0) - self.assertAlmostEqual(avpV_inst, 100., delta=1.0) + self.assertAlmostEqual(avpV_sim, 100., delta=1.) + self.assertAlmostEqual(avpV_inst, 100., delta=1.) def test_negative_volume(self): """Test for NpT with bad parameters.""" diff --git a/testsuite/python/scafacos_interface.py b/testsuite/python/scafacos_interface.py index 6977a8a0579..5c0776a7d00 100644 --- a/testsuite/python/scafacos_interface.py +++ b/testsuite/python/scafacos_interface.py @@ -350,10 +350,8 @@ def fcs_data(self): new_torques = np.copy(system.part.all().torque_lab) self.assertAlmostEqual(new_E_coulomb, ref_E_coulomb, delta=0) self.assertAlmostEqual(new_E_dipoles, ref_E_dipoles, delta=0) - np.testing.assert_allclose( - new_forces, ref_forces, atol=1e-10, rtol=1e-10) - np.testing.assert_allclose( - new_torques, ref_torques, atol=1e-10, rtol=1e-10) + np.testing.assert_allclose(new_forces, ref_forces, atol=0, rtol=0.) + np.testing.assert_allclose(new_torques, ref_torques, atol=0, rtol=0.) self.system.electrostatics.clear() self.system.magnetostatics.clear() diff --git a/testsuite/python/unittest_decorators.py b/testsuite/python/unittest_decorators.py index 44f1fa9b664..32b988b80cf 100644 --- a/testsuite/python/unittest_decorators.py +++ b/testsuite/python/unittest_decorators.py @@ -79,10 +79,3 @@ def skipIfUnmetModuleVersionRequirement(module, version_requirement): return unittest.skip( "Skipping test: version requirement not met for module {}".format(module)) return no_skip - - -def skipIfExistingFeatures(*args): - """Unittest skipIf decorator for existing Espresso features.""" - if espressomd.has_features(*args): - return unittest.skip("Skipping test: existing feature") - return no_skip From c4406fb3783e3cc82833512653947ac32fa9e46e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Wed, 6 Aug 2025 16:57:42 +0200 Subject: [PATCH 83/94] Fix regressions --- src/core/cell_system/CellStructure.hpp | 6 +++--- src/core/system/System.cpp | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 85963c81c69..6bcaa474865 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -457,7 +457,7 @@ struct CellStructure : public System::Leaf { #ifdef SHARED_MEMORY_PARALLELISM int get_cached_max_local_particle_id() const { return m_cached_max_local_particle_id; - }; + } #endif /** @@ -748,8 +748,8 @@ struct CellStructure : public System::Leaf { #ifdef NPT VirialType &get_local_virial() { return *m_local_virial; } #endif - AoSoA_pack &get_aosoa_data() { return *m_aosoa; }; - ListType &get_cabana_verlet_list() { return *m_cabana_verlet_list; }; + AoSoA_pack &get_aosoa_data() { return *m_aosoa; } + ListType &get_cabana_verlet_list() { return *m_cabana_verlet_list; } std::vector &get_unique_particles() { return m_unique_particles; } void set_index_map(); diff --git a/src/core/system/System.cpp b/src/core/system/System.cpp index 65b2e792e9c..c96819568a4 100644 --- a/src/core/system/System.cpp +++ b/src/core/system/System.cpp @@ -432,9 +432,6 @@ bool System::long_range_interactions_sanity_checks() const { } double System::get_interaction_range() const { - if (maximal_cutoff() > 1000000.) { - auto const max_cut = maximal_cutoff(); - } auto const max_cut = maximal_cutoff(); auto const verlet_skin = cell_structure->get_verlet_skin(); /* Consider skin only if there are actually interactions */ From b78db8ea83994893f5db366e2f4153db5f54f5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Wed, 6 Aug 2025 22:16:13 +0200 Subject: [PATCH 84/94] Fix regressions --- src/core/cell_system/CellStructure.cpp | 3 +++ src/core/forces.cpp | 7 +++++++ src/core/short_range_cabana.hpp | 10 ---------- testsuite/python/caliper.py | 8 +++++--- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 939150f61ee..5e81f67e081 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -103,6 +103,9 @@ static auto estimate_max_counts(int max_prefactor, double pair_cutoff, if (std::isinf(pair_cutoff)) { return number_of_unique_particles; } + if (pair_cutoff < 0.) { + pair_cutoff = 0.; + } auto const volume = Utils::int_pow<3>(pair_cutoff); auto max_counts = static_cast( std::ceil(static_cast(max_prefactor) * volume)); diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 072bb02d8cf..d2158436ac1 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -222,6 +222,9 @@ void System::System::calculate_forces() { }; #ifdef SHARED_MEMORY_PARALLELISM +#ifdef CALIPER + CALI_MARK_BEGIN("parallel short range"); +#endif auto const &verlet_criterion = VerletCriterion<>{*this, cell_structure->get_verlet_skin(), @@ -328,6 +331,10 @@ void System::System::calculate_forces() { cell_structure->non_bonded_loop(collision_kernel, verlet_criterion); #endif +#ifdef CALIPER + CALI_MARK_END("parallel short range"); +#endif + #else // SHARED_MEMORY_PARALLELISM auto pair_kernel = [coulomb_kernel_ptr = get_ptr(coulomb_kernel), diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 58c2a848f94..7f1eb0faa6d 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -177,21 +177,11 @@ void cabana_short_range(BondKernel const &bond_kernel, double bond_cutoff, ParticleRange const &particles, ParticleRange const &ghost_particles, VerletCriterion const &verlet_criterion = {}) { -#ifdef CALIPER - CALI_CXX_MARK_FUNCTION; -#endif - -#ifdef CALIPER - CALI_MARK_BEGIN("Espresso - Bond Kernel"); -#endif assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); if (bond_cutoff >= 0.) { cell_structure.bond_loop(bond_kernel); } -#ifdef CALIPER - CALI_MARK_END("Espresso - Bond Kernel"); -#endif // Cabana short range loop if (pair_cutoff > 0.) { diff --git a/testsuite/python/caliper.py b/testsuite/python/caliper.py index 265e2f741e7..a7dc44784cb 100644 --- a/testsuite/python/caliper.py +++ b/testsuite/python/caliper.py @@ -25,21 +25,23 @@ import sys import os -EXPECTED_LABELS = """ +EXPECTED_LABELS = f""" integrate Initial Force Calculation calculate_forces copy_particles_to_GPU init_forces_and_thermost calc_long_range_forces - short_range_loop + {'parallel short range' if espressomd.has_features( + ["SHARED_MEMORY_PARALLELISM"]) else 'short_range_loop'} copy_forces_from_GPU Integration loop calculate_forces copy_particles_to_GPU init_forces_and_thermost calc_long_range_forces - short_range_loop + {'parallel short range' if espressomd.has_features( + ["SHARED_MEMORY_PARALLELISM"]) else 'short_range_loop'} copy_forces_from_GPU calc_energies short_range_loop From 082d467502e3d6bda05c130740cebfe1f3fd7033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Thu, 7 Aug 2025 13:23:25 +0200 Subject: [PATCH 85/94] Avoid copies --- CMakeLists.txt | 1 - src/core/BoxGeometry.hpp | 4 +- src/core/forces.cpp | 77 ++++++++++---------------- src/core/forces_inline.hpp | 4 +- src/core/integrate.cpp | 2 - src/core/short_range_cabana.hpp | 97 +++++++++++++++------------------ 6 files changed, 77 insertions(+), 108 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eeeb6c121bd..1de1439f365 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -949,7 +949,6 @@ if(ESPRESSO_BUILD_WITH_CALIPER) set(CALIPER_WITH_MPI on CACHE BOOL "") set(CALIPER_WITH_NVTX off CACHE BOOL "") set(CALIPER_WITH_CUPTI off CACHE BOOL "") - # set(CALIPER_WITH_OMPT on CACHE BOOL "") set(CALIPER_INSTALL_CONFIG off CACHE BOOL "") set(CALIPER_INSTALL_HEADERS off CACHE BOOL "") set(BUILD_SHARED_LIBS ON) diff --git a/src/core/BoxGeometry.hpp b/src/core/BoxGeometry.hpp index aad3fcedc67..fdddb16161d 100644 --- a/src/core/BoxGeometry.hpp +++ b/src/core/BoxGeometry.hpp @@ -234,8 +234,8 @@ class BoxGeometry { * @param a1 y element of the terminal point. * @param a2 z element of the terminal point. * @param b0 x element of the initial point. - * @param b1 x element of the initial point. - * @param b2 x element of the initial point. + * @param b1 y element of the initial point. + * @param b2 z element of the initial point. * @return Vector from @p b to @p a that minimizes the distance across * periodic images, i.e. a - b. */ diff --git a/src/core/forces.cpp b/src/core/forces.cpp index d2158436ac1..46b13b99dc6 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -221,27 +221,26 @@ void System::System::calculate_forces() { box_geo, coulomb_kernel_ptr); }; + VerletCriterion<> const verlet_criterion{*this, + cell_structure->get_verlet_skin(), + get_interaction_range(), + coulomb_cutoff, + dipole_cutoff, + collision_detection_cutoff}; + #ifdef SHARED_MEMORY_PARALLELISM #ifdef CALIPER CALI_MARK_BEGIN("parallel short range"); #endif - auto const &verlet_criterion = - VerletCriterion<>{*this, - cell_structure->get_verlet_skin(), - get_interaction_range(), - coulomb_cutoff, - dipole_cutoff, - collision_detection_cutoff}; - update_cabana_state(*cell_structure, particles, - cell_structure->ghost_particles(), verlet_criterion, + update_cabana_state(*cell_structure, verlet_criterion, get_interaction_range()); - auto unique_particles = cell_structure->get_unique_particles(); - auto local_force = cell_structure->get_local_force(); + auto &unique_particles = cell_structure->get_unique_particles(); + auto &local_force = cell_structure->get_local_force(); #ifdef ROTATION - auto local_torque = cell_structure->get_local_torque(); + auto &local_torque = cell_structure->get_local_torque(); #endif #ifdef NPT - auto local_virial = cell_structure->get_local_virial(); + auto &local_virial = cell_structure->get_local_virial(); #endif auto const &aosoa = cell_structure->get_aosoa_data(); @@ -265,9 +264,7 @@ void System::System::calculate_forces() { aosoa); cabana_short_range(bond_kernel, first_neighbor_kernel, *cell_structure, - get_interaction_range(), bonded_ias->maximal_cutoff(), - particles, cell_structure->ghost_particles(), - verlet_criterion); + get_interaction_range(), bonded_ias->maximal_cutoff()); // Force and Torque reduction int num_threads = execution_space().concurrency(); Kokkos::RangePolicy policy(0, unique_particles.size()); @@ -277,47 +274,35 @@ void System::System::calculate_forces() { &local_torque, #endif &unique_particles, num_threads](const int i) { - double fx = 0.; - double fy = 0.; - double fz = 0.; + Utils::Vector3d force{}; #ifdef ROTATION - double tx = 0.; - double ty = 0.; - double tz = 0.; + Utils::Vector3d torque{}; #endif for (int tid = 0; tid < num_threads; ++tid) { - fx += local_force(i, tid, 0); - fy += local_force(i, tid, 1); - fz += local_force(i, tid, 2); + force[0] += local_force(i, tid, 0); + force[1] += local_force(i, tid, 1); + force[2] += local_force(i, tid, 2); #ifdef ROTATION - tx += local_torque(i, tid, 0); - ty += local_torque(i, tid, 1); - tz += local_torque(i, tid, 2); + torque[0] += local_torque(i, tid, 0); + torque[1] += local_torque(i, tid, 1); + torque[2] += local_torque(i, tid, 2); #endif } - // auto &p = unique_particles.at(i); - // p->force() += Utils::Vector3d{fx, fy, fz}; - unique_particles.at(i)->force() += - Utils::Vector3d{fx, fy, fz}; + unique_particles.at(i)->force() += force; #ifdef ROTATION - // p->torque() += Utils::Vector3d{tx, ty, tz}; - unique_particles.at(i)->torque() += - Utils::Vector3d{tx, ty, tz}; + unique_particles.at(i)->torque() += torque; #endif }); Kokkos::fence(); #ifdef NPT - double vx = 0.; - double vy = 0.; - double vz = 0.; + Utils::Vector3d virial{}; for (int tid = 0; tid < num_threads; ++tid) { - vx += local_virial(tid, 0); - vy += local_virial(tid, 1); - vz += local_virial(tid, 2); + virial[0] += local_virial(tid, 0); + virial[1] += local_virial(tid, 1); + virial[2] += local_virial(tid, 2); } - Utils::Vector3d virial_vec{vx, vy, vz}; - npt_add_virial_force_contribution(virial_vec); + npt_add_virial_force_contribution(virial); #endif #ifdef COLLISION_DETECTION @@ -361,11 +346,7 @@ void System::System::calculate_forces() { }; short_range_loop(bond_kernel, pair_kernel, *cell_structure, maximal_cutoff(), - bonded_ias->maximal_cutoff(), - VerletCriterion<>{*this, cell_structure->get_verlet_skin(), - get_interaction_range(), coulomb_cutoff, - dipole_cutoff, - collision_detection_cutoff}); + bonded_ias->maximal_cutoff(), verlet_criterion); #endif // SHARED_MEMORY_PARALLELISM constraints->add_forces(particles, get_sim_time()); diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 64ca1f98503..7a46a1a9a6d 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -196,7 +196,7 @@ inline void add_non_bonded_pair_without_p( } /***********************************************/ - /* short range cloumb potentials */ + /* short-range electrostatics */ /***********************************************/ #ifdef ELECTROSTATICS @@ -208,7 +208,7 @@ inline void add_non_bonded_pair_without_p( } /** - * For the interaction which need particle information + * @brief For interactions which need particle information. */ inline void add_non_bonded_pair_force_with_p( Particle &p1, Particle &p2, ParticleForce &pf, diff --git a/src/core/integrate.cpp b/src/core/integrate.cpp index fd8bb304ee8..8106d42196e 100644 --- a/src/core/integrate.cpp +++ b/src/core/integrate.cpp @@ -506,13 +506,11 @@ int System::System::integrate(int n_steps, int reuse_forces) { lb_active = lb.is_solver_set(); ek_active = ek.is_ready_for_propagation(); #ifdef SHARED_MEMORY_PARALLELISM - // cell_structure->set_steepest_descent_flag(false); cell_structure->set_max_prefactor(5); #endif } #ifdef SHARED_MEMORY_PARALLELISM else { - // cell_structure->set_steepest_descent_flag(true); cell_structure->set_max_prefactor(8); } #endif diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index 7f1eb0faa6d..d54419ae094 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -46,34 +46,37 @@ inline void write_particle(Particle const &p, int const &id, } } -template ESPRESSO_ATTR_ALWAYS_INLINE inline void construct_verlet_list( - CellStructure &cell_structure, VerletCriterion const &verlet_criterion, + CellStructure &cell_structure, auto const &verlet_criterion, Kokkos::View const &id_to_index, const int max_id) { auto const &cells = std::as_const(cell_structure).decomposition().local_cells(); auto const distance_function = detail::MinimalImageDistance{ std::as_const(cell_structure).decomposition().box()}; - auto verlet_list = cell_structure.get_cabana_verlet_list(); + auto &verlet_list = cell_structure.get_cabana_verlet_list(); + + // implementation detail: max_id refers to the max local particle id, + // but ghost particles from other ranks may have larger particle ids; + // in addition, -1 is used as a sentinel value for particle ids auto intra_kernel = [&cells, &distance_function, &verlet_criterion, &id_to_index, &verlet_list, max_id](const int i) { auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto const &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs in this cell */ - for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { - if ((*jt).id() > max_id) - continue; - if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { - int jj = id_to_index((*jt).id()); - if (jj >= 0) { - verlet_list.addNeighborLB(ii, jj); + if (p1.id() <= max_id) { + auto const ii = id_to_index(p1.id()); + if (ii >= 0) { + // pairs in this cell + for (auto jt = std::next(it); jt != local_particles.end(); ++jt) { + if ((*jt).id() <= max_id) { + if (verlet_criterion(p1, *jt, distance_function(p1, *jt))) { + auto const jj = id_to_index((*jt).id()); + if (jj >= 0) { + verlet_list.addNeighborLB(ii, jj); + } + } + } } } } @@ -85,20 +88,20 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void construct_verlet_list( auto &local_particles = cells[i]->particles(); for (auto it = local_particles.begin(); it != local_particles.end(); ++it) { auto const &p1 = *it; - if (p1.id() > max_id) - continue; - int ii = id_to_index(p1.id()); - if (ii < 0) - continue; - /* Pairs with neighbors */ - for (auto &neighbor : cells[i]->neighbors().red()) { - for (auto const &p2 : neighbor->particles()) { - if (p2.id() > max_id) - continue; - if (verlet_criterion(p1, p2, distance_function(p1, p2))) { - int jj = id_to_index(p2.id()); - if (jj >= 0) { - verlet_list.addNeighbor(ii, jj); + if (p1.id() <= max_id) { + auto const ii = id_to_index(p1.id()); + if (ii >= 0) { + // pairs with neighboring cells + for (auto &neighbor : cells[i]->neighbors().red()) { + for (auto const &p2 : neighbor->particles()) { + if (p2.id() <= max_id) { + if (verlet_criterion(p1, p2, distance_function(p1, p2))) { + auto const jj = id_to_index(p2.id()); + if (jj >= 0) { + verlet_list.addNeighbor(ii, jj); + } + } + } } } } @@ -113,22 +116,18 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void construct_verlet_list( Kokkos::fence(); } -template -ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( - CellStructure &cell_structure, ParticleRange const &particles, - ParticleRange const &ghost_particles, - VerletCriterion const &verlet_criterion, double const pair_cutoff) { - // Number of threads - int num_threads = execution_space().concurrency(); +ESPRESSO_ATTR_ALWAYS_INLINE inline void +update_cabana_state(CellStructure &cell_structure, auto const &verlet_criterion, + double const pair_cutoff) { - bool const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or + int num_threads = execution_space().concurrency(); + auto const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or (not cell_structure.use_verlet_list); if (rebuild) { // If we have to rebuild, we need to count the particles cell_structure.set_index_map(); // parallelized index_map - - // Create essential variable for MD + // Create essential variables for MD cell_structure.rebuild_local_properties(num_threads, pair_cutoff); } else { // If we do not rebuild we can use the saved map @@ -136,9 +135,9 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( } auto const unique_particles = cell_structure.get_unique_particles(); auto aosoa = cell_structure.get_aosoa_data(); - int max_id = cell_structure.get_cached_max_local_particle_id(); + auto max_id = cell_structure.get_cached_max_local_particle_id(); - // Fill the essential variable for MD + // Fill the essential variables for MD { // =================================================== // Fill particle storage @@ -160,7 +159,6 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( // Get Verlet Pairs and Fill Verlet list // =================================================== - // Rebuild verlet list if needed if (rebuild) { construct_verlet_list(cell_structure, verlet_criterion, id_to_index, max_id); @@ -169,14 +167,9 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state( } } -template -void cabana_short_range(BondKernel const &bond_kernel, - PairKernel const &forces_kernel, +void cabana_short_range(auto const &bond_kernel, auto const &forces_kernel, CellStructure &cell_structure, double pair_cutoff, - double bond_cutoff, ParticleRange const &particles, - ParticleRange const &ghost_particles, - VerletCriterion const &verlet_criterion = {}) { + double bond_cutoff) { assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); if (bond_cutoff >= 0.) { @@ -185,13 +178,11 @@ void cabana_short_range(BondKernel const &bond_kernel, // Cabana short range loop if (pair_cutoff > 0.) { - auto cabana_verlet_list = cell_structure.get_cabana_verlet_list(); - // cabana_verlet_list.get_variance_max_counts(); + auto &cabana_verlet_list = cell_structure.get_cabana_verlet_list(); Kokkos::RangePolicy policy( 0, cell_structure.get_unique_particles().size()); Cabana::neighbor_parallel_for(policy, forces_kernel, cabana_verlet_list, Cabana::FirstNeighborsTag(), - // Cabana::TeamOpTag()); Cabana::SerialOpTag()); Kokkos::fence(); } From b6492c99306da9647cf37ee3132ea37430b70428 Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 7 Aug 2025 19:54:23 +0200 Subject: [PATCH 86/94] Fixed bug for NPT with P3M --- src/core/forces_cabana.hpp | 16 ++++++-- src/core/forces_inline.hpp | 14 ++++++- testsuite/python/integrator_npt_stats.py | 49 ++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index 40b1c18bb1b..c63f6139912 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -118,8 +118,6 @@ struct ForcesKernel { aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); auto const dist = d.norm(); - auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); - #if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) auto &p1 = *unique_particles.at(i); auto &p2 = *unique_particles.at(j); @@ -128,13 +126,23 @@ struct ForcesKernel { #ifdef EXCLUSIONS auto const do_nonbonded_flag = do_nonbonded(p1, p2); #else +#if defined(LONG_RANGE_KERNELS) auto constexpr do_nonbonded_flag = true; +#endif //LONG_RANGE_KERNELS #endif - add_non_bonded_pair_without_p(pf, d, dist, q1q2, ia_params, - do_nonbonded_flag, coulomb_kernel); + if (dist < ia_params.max_cut) { +#ifdef EXCLUSIONS + if (do_nonbonded_flag) { +#endif + pf += calc_central_radial_force(ia_params, d, dist); +#ifdef EXCLUSIONS + } +#endif + } #if defined(LONG_RANGE_KERNELS) + auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); add_non_bonded_pair_force_with_p(p1, p2, pf, #ifdef NPT virial, diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 7a46a1a9a6d..4e8fb3c3279 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -262,6 +262,7 @@ inline void add_non_bonded_pair_force_with_p( #ifdef ELECTROSTATICS // real-space electrostatic charge-charge interaction if (q1q2 != 0. and coulomb_kernel != nullptr) { + pf.f += (*coulomb_kernel)(q1q2, d, dist); #ifdef NPT #ifdef SHARED_MEMORY_PARALLELISM virial[0] += (*coulomb_u_kernel)(p1, p2, q1q2, d, dist); @@ -338,11 +339,20 @@ inline auto add_non_bonded_pair_force( #ifdef EXCLUSIONS auto const do_nonbonded_flag = do_nonbonded(p1, p2); #else +#if defined(LONG_RANGE_KERNELS) auto constexpr do_nonbonded_flag = true; +#endif //LONG_RANGE_KERNELS #endif - add_non_bonded_pair_without_p(pf, d, dist, q1q2, ia_params, do_nonbonded_flag, - coulomb_kernel); + if (dist < ia_params.max_cut) { +#ifdef EXCLUSIONS + if (do_nonbonded_flag) { +#endif + pf += calc_central_radial_force(ia_params, d, dist); +#ifdef EXCLUSIONS + } +#endif + } #if defined(LONG_RANGE_KERNELS) add_non_bonded_pair_force_with_p( diff --git a/testsuite/python/integrator_npt_stats.py b/testsuite/python/integrator_npt_stats.py index ed352fe1a1c..95f8af753a1 100644 --- a/testsuite/python/integrator_npt_stats.py +++ b/testsuite/python/integrator_npt_stats.py @@ -116,6 +116,55 @@ def test_compressibility_and_pressure(self): self.assertAlmostEqual(avpV_sim, 100., delta=1.) self.assertAlmostEqual(avpV_inst, 100., delta=1.) + @utx.skipIfMissingFeatures("WCA", "P3M") + def test_pressure_compared_to_instantaneous_withP3M(self): + """Test for Npt with P3M.""" + + data = np.genfromtxt(tests_common.data_path("npt_lj_system.data")) + ref_box_l = np.max(data[:, 0:3]) + + system = self.system + system.box_l = 3 * [ref_box_l] + system.non_bonded_inter[2, 2].wca.set_params(epsilon=1., sigma=1.) + p3m = espressomd.electrostatics.P3M( + prefactor=2.0, accuracy=1e-2, mesh=3 * [14], cao=5, tune=True) + dt = 0.01 + system.time_step = dt + + direction = [True] * 3 + p_ext = 1.0 + system.box_l = 3 * [ref_box_l] + system.part.add(pos=data[:, 0:3], type=len(data) * [2]) + system.part.all().pos = data[:, 0:3] + system.part.all().v = data[:, 3:6] + system.part.all().q = np.sign(np.arange(100) - 50 + 0.5) + self.system.integrator.set_vv() + self.system.electrostatics.solver = p3m + + if self.barostat == "Andersen": + system.thermostat.set_npt(kT=1.0, gamma0=0.2, gammav=0.01, seed=42) + system.integrator.set_isotropic_npt( + ext_pressure=p_ext, piston=0.0001) + else: + system.thermostat.set_npt( + kT=1.0, gamma0=0.5, gammav=0.001, seed=42) + system.integrator.set_isotropic_npt( + ext_pressure=p_ext, piston=4.0, barostat=self.barostat) + + steps = int(0.1/dt) + + for n in range(100): + system.integrator.run(steps) + p_sim = system.analysis.pressure()['total'] + p_kin = system.analysis.pressure()['kinetic'] + #virial of electrostatic force from system.analysis + p_vir = p_sim - p_kin + #virial of electrostatic force from instantaneous_pressure + p_inst_vir = system.analysis.get_instantaneous_pressure_virial() + + np.testing.assert_allclose(p_vir, p_inst_vir, atol=1e-2) + + def test_negative_volume(self): """Test for NpT with bad parameters.""" From 63477c1f3502145cf16f612e38898fd31ab67ffe Mon Sep 17 00:00:00 2001 From: Hideki Kobayashi Date: Thu, 7 Aug 2025 19:58:29 +0200 Subject: [PATCH 87/94] Formatting --- src/core/forces_cabana.hpp | 4 ++-- src/core/forces_inline.hpp | 2 +- testsuite/python/integrator_npt_stats.py | 10 ++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index c63f6139912..6d67e76ccb1 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -128,14 +128,14 @@ struct ForcesKernel { #else #if defined(LONG_RANGE_KERNELS) auto constexpr do_nonbonded_flag = true; -#endif //LONG_RANGE_KERNELS +#endif // LONG_RANGE_KERNELS #endif if (dist < ia_params.max_cut) { #ifdef EXCLUSIONS if (do_nonbonded_flag) { #endif - pf += calc_central_radial_force(ia_params, d, dist); + pf += calc_central_radial_force(ia_params, d, dist); #ifdef EXCLUSIONS } #endif diff --git a/src/core/forces_inline.hpp b/src/core/forces_inline.hpp index 4e8fb3c3279..381a1934347 100644 --- a/src/core/forces_inline.hpp +++ b/src/core/forces_inline.hpp @@ -341,7 +341,7 @@ inline auto add_non_bonded_pair_force( #else #if defined(LONG_RANGE_KERNELS) auto constexpr do_nonbonded_flag = true; -#endif //LONG_RANGE_KERNELS +#endif // LONG_RANGE_KERNELS #endif if (dist < ia_params.max_cut) { diff --git a/testsuite/python/integrator_npt_stats.py b/testsuite/python/integrator_npt_stats.py index 95f8af753a1..9e3572d3d9a 100644 --- a/testsuite/python/integrator_npt_stats.py +++ b/testsuite/python/integrator_npt_stats.py @@ -131,7 +131,6 @@ def test_pressure_compared_to_instantaneous_withP3M(self): dt = 0.01 system.time_step = dt - direction = [True] * 3 p_ext = 1.0 system.box_l = 3 * [ref_box_l] system.part.add(pos=data[:, 0:3], type=len(data) * [2]) @@ -151,20 +150,19 @@ def test_pressure_compared_to_instantaneous_withP3M(self): system.integrator.set_isotropic_npt( ext_pressure=p_ext, piston=4.0, barostat=self.barostat) - steps = int(0.1/dt) + steps = int(0.1 / dt) - for n in range(100): + for _ in range(100): system.integrator.run(steps) p_sim = system.analysis.pressure()['total'] p_kin = system.analysis.pressure()['kinetic'] - #virial of electrostatic force from system.analysis + # virial of electrostatic force from system.analysis p_vir = p_sim - p_kin - #virial of electrostatic force from instantaneous_pressure + # virial of electrostatic force from instantaneous_pressure p_inst_vir = system.analysis.get_instantaneous_pressure_virial() np.testing.assert_allclose(p_vir, p_inst_vir, atol=1e-2) - def test_negative_volume(self): """Test for NpT with bad parameters.""" From 2a55a71f28b806162e70a67a12c6222dc101f984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 8 Aug 2025 18:27:48 +0200 Subject: [PATCH 88/94] Apply SOLID principles --- src/core/aosoa_pack.hpp | 17 ++- src/core/cell_system/CellStructure.cpp | 17 +-- src/core/cell_system/CellStructure.hpp | 122 +++++++++++++--------- src/core/communication.cpp | 6 +- src/core/communication.hpp | 13 +-- src/core/custom_verlet_list.hpp | 105 +++++++------------ src/core/forces.cpp | 11 +- src/core/forces_cabana.hpp | 54 +++++----- src/core/short_range_cabana.hpp | 113 ++++++++++---------- src/core/system/System.hpp | 2 +- testsuite/python/integrator_exceptions.py | 38 +++++++ testsuite/python/integrator_npt_stats.py | 86 +-------------- testsuite/python/npt_thermostat.py | 46 ++++++++ 13 files changed, 309 insertions(+), 321 deletions(-) diff --git a/src/core/aosoa_pack.hpp b/src/core/aosoa_pack.hpp index e3ba514d9e3..3215fb01f88 100644 --- a/src/core/aosoa_pack.hpp +++ b/src/core/aosoa_pack.hpp @@ -22,21 +22,20 @@ #ifdef SHARED_MEMORY_PARALLELISM #include "cell_system/CellStructure.hpp" -#include -using execution_space = Kokkos::DefaultExecutionSpace; +#include -struct AoSoA_pack { - AoSoAType::member_slice_type<0> position; - AoSoAType::member_slice_type<1> charge; - AoSoAType::member_slice_type<2> id; - AoSoAType::member_slice_type<3> type; +struct CellStructure::AoSoA_pack { + CellStructure::AoSoAType::member_slice_type<0> position; + CellStructure::AoSoAType::member_slice_type<1> charge; + CellStructure::AoSoAType::member_slice_type<2> id; + CellStructure::AoSoAType::member_slice_type<3> type; AoSoA_pack() = default; - AoSoA_pack(AoSoAType &aosoa) + AoSoA_pack(CellStructure::AoSoAType &aosoa) : position(Cabana::slice<0>(aosoa)), charge(Cabana::slice<1>(aosoa)), id(Cabana::slice<2>(aosoa)), type(Cabana::slice<3>(aosoa)) {} }; -#endif +#endif // SHARED_MEMORY_PARALLELISM diff --git a/src/core/cell_system/CellStructure.cpp b/src/core/cell_system/CellStructure.cpp index 5e81f67e081..2e477079ba3 100644 --- a/src/core/cell_system/CellStructure.cpp +++ b/src/core/cell_system/CellStructure.cpp @@ -86,15 +86,18 @@ CellStructure::~CellStructure() { if (m_particle_storage) { m_particle_storage.reset(); } - if (m_cabana_verlet_list) { - m_cabana_verlet_list.reset(); + if (m_verlet_list_cabana) { + m_verlet_list_cabana.reset(); } + // Kokkos handle can be freed after all Cabana containers have been freed + m_kokkos_handle.reset(); #endif } #ifdef SHARED_MEMORY_PARALLELISM -void CellStructure::set_kokkos_handle(std::shared_ptr handle) { +void CellStructure::set_kokkos_handle( + std::shared_ptr handle) { m_kokkos_handle = std::move(handle); } @@ -138,8 +141,8 @@ void CellStructure::rebuild_local_properties(std::size_t const num_threads, // particle properties are defined in aosoa_pack.hpp m_aosoa = std::make_unique(*m_particle_storage); - auto max_counts = estimate_max_counts(max_prefactor, pair_cutoff, num_part); - m_cabana_verlet_list = std::make_unique(0ul, num_part, max_counts); + auto max_counts = estimate_max_counts(m_max_prefactor, pair_cutoff, num_part); + m_verlet_list_cabana = std::make_unique(0ul, num_part, max_counts); } void CellStructure::reset_local_properties() { @@ -365,7 +368,7 @@ void CellStructure::resort_particles(bool global_flag) { auto const &lebc = get_system().box_geo->lees_edwards_bc(); m_rebuild_verlet_list = true; - m_rebuild_cabana_verlet_list = true; + m_rebuild_verlet_list_cabana = true; m_le_pos_offset_at_last_resort = lebc.pos_offset; #ifdef ADDITIONAL_CHECKS @@ -415,7 +418,7 @@ void CellStructure::set_verlet_skin(double value) { assert(value >= 0.); m_verlet_skin = value; m_verlet_skin_set = true; - m_rebuild_cabana_verlet_list = true; + m_rebuild_verlet_list_cabana = true; get_system().on_verlet_skin_change(); } diff --git a/src/core/cell_system/CellStructure.hpp b/src/core/cell_system/CellStructure.hpp index 6bcaa474865..32b8eaa5ae9 100644 --- a/src/core/cell_system/CellStructure.hpp +++ b/src/core/cell_system/CellStructure.hpp @@ -60,7 +60,7 @@ #include #endif -// forward declaration to not have to import cabana +// forward declarations #ifdef SHARED_MEMORY_PARALLELISM namespace Kokkos { template class View; @@ -72,28 +72,16 @@ namespace Cabana { class HalfNeighborTag; struct VerletLayout2D; class TeamVectorOpTag; -template -class CustomVerletList; template struct MemberTypes; template class AoSoA; } // namespace Cabana -struct AoSoA_pack; +namespace Communication { struct KokkosHandle; -// To construct AoSoA, vector_length is defined HERE. -const int vector_length = 1; - -using ForceType = Kokkos::View; -using VirialType = Kokkos::View; -using data_types = Cabana::MemberTypes; -using memory_space = Kokkos::HostSpace; -using AoSoAType = Cabana::AoSoA>; -using ListAlgorithm = Cabana::HalfNeighborTag; -using ListType = - Cabana::CustomVerletList; -#endif +} // namespace Communication +template +class CustomVerletList; +#endif // SHARED_MEMORY_PARALLELISM template concept ParticleCallback = requires(Callable c, Particle &p) { @@ -180,7 +168,23 @@ struct EuclidianDistance { * system which are not common between different cell systems have to * be stored in separate structures. */ -struct CellStructure : public System::Leaf { +class CellStructure : public System::Leaf { +#ifdef SHARED_MEMORY_PARALLELISM +public: + static constexpr auto vector_length = 1; + struct AoSoA_pack; + using ForceType = Kokkos::View; + using VirialType = Kokkos::View; + using data_types = Cabana::MemberTypes; + using memory_space = Kokkos::HostSpace; + using AoSoAType = Cabana::AoSoA>; + using ListAlgorithm = Cabana::HalfNeighborTag; + using ListType = + CustomVerletList; +#endif // SHARED_MEMORY_PARALLELISM + private: /** The local id-to-particle index */ std::vector m_particle_index; @@ -191,17 +195,18 @@ struct CellStructure : public System::Leaf { /** One of @ref Cells::Resort, announces the level of resort needed. */ unsigned m_resort_particles = Cells::RESORT_NONE; + bool m_verlet_skin_set = false; bool m_rebuild_verlet_list = true; - bool m_rebuild_cabana_verlet_list = true; + bool m_rebuild_verlet_list_cabana = true; std::vector> m_verlet_list; double m_le_pos_offset_at_last_resort = 0.; /** @brief Verlet list skin. */ double m_verlet_skin = 0.; - bool m_verlet_skin_set = false; double m_verlet_reuse = 0.; #ifdef SHARED_MEMORY_PARALLELISM - int m_cached_max_local_particle_id; - + int m_cached_max_local_particle_id = 0; + int m_max_prefactor = 8; + int m_max_id = 0; std::unique_ptr m_local_force; #ifdef ROTATION std::unique_ptr m_local_torque; @@ -209,13 +214,13 @@ struct CellStructure : public System::Leaf { #ifdef NPT std::unique_ptr m_local_virial; #endif + std::unique_ptr m_verlet_list_cabana; std::unique_ptr m_particle_storage; - /** particle properties for Cabana defined in aosoa_pack.hpp */ + /** particle properties for Cabana */ std::unique_ptr m_aosoa; /** The local id-to-index for aosoa data */ std::vector m_unique_particles; - - std::unique_ptr m_cabana_verlet_list; + std::shared_ptr m_kokkos_handle; #endif // SHARED_MEMORY_PARALLELISM public: @@ -719,38 +724,55 @@ struct CellStructure : public System::Leaf { } #ifdef SHARED_MEMORY_PARALLELISM -private: - int max_prefactor = 8; - int m_max_id = 0; - std::shared_ptr m_kokkos_handle; - public: - bool get_rebuild_verlet_list() const { return m_rebuild_verlet_list; } - bool get_rebuild_cabana_verlet_list() const { - return m_rebuild_cabana_verlet_list; - } - void mark_rebuild_cabana_verlet_list_as_UpToDate() { - m_rebuild_cabana_verlet_list = false; - } + void set_max_prefactor(int value) { m_max_prefactor = value; } + auto get_max_id() const { return m_max_id; } - void set_max_prefactor(int value) { max_prefactor = value; } - - int get_max_id() { return m_max_id; } - - void set_kokkos_handle(std::shared_ptr handle); + void set_kokkos_handle(std::shared_ptr handle); void rebuild_local_properties(std::size_t num_threads, double pair_cutoff); void reset_local_properties(); - ForceType &get_local_force() { return *m_local_force; } + auto &get_local_force() { return *m_local_force; } #ifdef ROTATION - ForceType &get_local_torque() { return *m_local_torque; } + auto &get_local_torque() { return *m_local_torque; } #endif #ifdef NPT - VirialType &get_local_virial() { return *m_local_virial; } + auto &get_local_virial() { return *m_local_virial; } #endif - AoSoA_pack &get_aosoa_data() { return *m_aosoa; } - ListType &get_cabana_verlet_list() { return *m_cabana_verlet_list; } - std::vector &get_unique_particles() { return m_unique_particles; } + auto &get_aosoa() { return *m_aosoa; } + auto const &get_unique_particles() const { return m_unique_particles; } + auto const &get_verlet_list_cabana() const { return *m_verlet_list_cabana; } + + [[nodiscard]] auto is_verlet_list_cabana_rebuild_needed() const { + return m_rebuild_verlet_list_cabana or (not use_verlet_list); + } + + /** + * @brief Reset local properties of the Verlet list. + * @param n_threads Number of threads. + * @param cutoff Pair interaction cutoff. + * @return True if a rebuild is needed. + */ + [[nodiscard]] auto prepare_verlet_list_cabana(int n_threads, double cutoff) { + auto const rebuild = is_verlet_list_cabana_rebuild_needed(); + if (rebuild) { + // If we have to rebuild, we need to count the particles + set_index_map(); // parallelized index_map + // Create essential variables for MD + rebuild_local_properties(n_threads, cutoff); + } else { + // If we do not rebuild we can use the saved map + reset_local_properties(); + } + return rebuild; + } + + void rebuild_verlet_list_cabana(auto &&kernel) { + assert(is_verlet_list_cabana_rebuild_needed()); + kernel(m_decomposition->local_cells(), m_decomposition->box(), + *m_verlet_list_cabana); + m_rebuild_verlet_list_cabana = false; + } void set_index_map(); inline void set_index_map(ParticleRange const &particles, @@ -806,7 +828,7 @@ struct CellStructure : public System::Leaf { }); m_rebuild_verlet_list = false; - m_rebuild_cabana_verlet_list = true; + m_rebuild_verlet_list_cabana = true; } else { auto const maybe_box = decomposition().minimum_image_distance(); /* In this case the pair kernel is just run over the verlet list. */ diff --git a/src/core/communication.cpp b/src/core/communication.cpp index 8bfe287cbee..4fed2019173 100644 --- a/src/core/communication.cpp +++ b/src/core/communication.cpp @@ -32,8 +32,6 @@ #endif #ifdef SHARED_MEMORY_PARALLELISM -#include "cell_system/CellStructure.hpp" -#include "system/System.hpp" #include #include #include @@ -56,16 +54,18 @@ #include #ifdef SHARED_MEMORY_PARALLELISM +namespace Communication { struct KokkosHandle { KokkosHandle() { Kokkos::initialize(); } ~KokkosHandle() { Kokkos::finalize(); } }; +} // namespace Communication #endif boost::mpi::communicator comm_cart; Communicator communicator{}; #ifdef SHARED_MEMORY_PARALLELISM -std::shared_ptr kokkos_handle{}; +std::shared_ptr kokkos_handle{}; #endif namespace Communication { diff --git a/src/core/communication.hpp b/src/core/communication.hpp index 568d935b282..bd2dbb21477 100644 --- a/src/core/communication.hpp +++ b/src/core/communication.hpp @@ -60,8 +60,10 @@ extern int this_node; /** The communicator */ extern boost::mpi::communicator comm_cart; #ifdef SHARED_MEMORY_PARALLELISM +namespace Communication { struct KokkosHandle; -extern std::shared_ptr kokkos_handle; +} // namespace Communication +extern std::shared_ptr kokkos_handle; #endif struct Communicator { @@ -113,12 +115,3 @@ namespace Communication { void init(std::shared_ptr mpi_env); void deinit(); } // namespace Communication - -struct MpiContainerUnitTest { - std::shared_ptr m_mpi_env; - MpiContainerUnitTest(int argc, char **argv) { - m_mpi_env = mpi_init(argc, argv); - Communication::init(m_mpi_env); - } - ~MpiContainerUnitTest() { Communication::deinit(); } -}; diff --git a/src/core/custom_verlet_list.hpp b/src/core/custom_verlet_list.hpp index 22544f35be7..d68e50e7909 100644 --- a/src/core/custom_verlet_list.hpp +++ b/src/core/custom_verlet_list.hpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010-2022 The ESPResSo project + * Copyright (C) 2025 The ESPResSo project * * This file is part of ESPResSo. * @@ -16,6 +16,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ + #pragma once #ifdef SHARED_MEMORY_PARALLELISM @@ -23,29 +24,22 @@ #include #include +#include #include -namespace Cabana { // ONLY FOR 2D LAYOUT, OTHERWISE NEIGHBOR LIST INTERFACE IMPLEMENTATION WILL // CAUSE PROBLEMS (NOT IMPLEMENTED) template -class CustomVerletList - : public VerletList { + class BuildTag = Cabana::TeamVectorOpTag> +class CustomVerletList : public Cabana::VerletList { public: - using Base = VerletList; - - // Default constructor - CustomVerletList() : Base() {} - - // Custom constructor + CustomVerletList() = default; CustomVerletList(std::size_t const begin, std::size_t const end, std::size_t const max_neigh) { initializeData(end - begin, max_neigh); } - virtual ~CustomVerletList() {}; -public: Kokkos::View counts; Kokkos::View neighbors; @@ -62,35 +56,23 @@ class CustomVerletList // Method to add a neighbor KOKKOS_INLINE_FUNCTION void addNeighborAtomicLB(int pid, int nid) { - std::size_t count = counts(pid); - std::size_t count_n = counts(nid); + auto count = counts(pid); + auto count_n = counts(nid); if (count > count_n) { - int tmp = pid; - pid = nid; - nid = tmp; + std::swap(pid, nid); } count = Kokkos::atomic_fetch_add(&counts(pid), 1); -#ifndef NDEBUG - if (count >= neighbors.extent(1)) { - throw std::runtime_error( - "Number of count is larger than VerletList size."); - } -#endif + assert(count < neighbors.extent(1)); neighbors(pid, count) = nid; } // Thread safe but non atomic method to add a neighbor KOKKOS_INLINE_FUNCTION void addNeighbor(int pid, int nid) { - std::size_t count = counts(pid); + auto const count = counts(pid); -#ifndef NDEBUG - if (count >= neighbors.extent(1)) { - throw std::runtime_error( - "Number of count is larger than VerletList size."); - } -#endif + assert(count < neighbors.extent(1)); neighbors(pid, count) = nid; counts(pid) += 1; } @@ -98,21 +80,14 @@ class CustomVerletList // Non atomic and load balancing method to add a neighbor KOKKOS_INLINE_FUNCTION void addNeighborLB(int pid, int nid) { - std::size_t count = counts(pid); - std::size_t count_n = counts(nid); + auto count = counts(pid); + auto count_n = counts(nid); if (count > count_n) { - int tmp = pid; - pid = nid; - nid = tmp; + std::swap(pid, nid); count = counts(pid); } -#ifndef NDEBUG - if (count >= neighbors.extent(1)) { - throw std::runtime_error( - "Number of count is larger than VerletList size."); - } -#endif + assert(count < neighbors.extent(1)); neighbors(pid, count) = nid; counts(pid) += 1; } @@ -121,11 +96,11 @@ class CustomVerletList KOKKOS_INLINE_FUNCTION void sortNeighbors() { Kokkos::parallel_for( - "custom_velet_list::sort_neighbors", + "custom_verlet_list::sort_neighbors", Kokkos::RangePolicy(0, counts.size()), [&](const int i) { const int count = counts(i); - int *ptr = &neighbors(i, 0); + auto *ptr = &neighbors(i, 0); std::sort(ptr, ptr + count); }); Kokkos::fence(); @@ -133,33 +108,33 @@ class CustomVerletList // Find max counts KOKKOS_INLINE_FUNCTION - std::size_t get_variance_max_counts() { - std::size_t max_counts = 0; - std::size_t ave_counts = 0; - std::size_t ave_sq_counts = 0; + auto get_variance_max_counts(auto &ostream) { + auto max_counts = 0l; + auto ave_counts = 0l; + auto ave_sq_counts = 0l; for (int pid = 0; pid < counts.extent(0); ++pid) { - std::size_t count = counts(pid); + auto const count = static_cast(counts(pid)); if (max_counts < count) max_counts = count; ave_counts += count; ave_sq_counts += count * count; } if (counts.extent(0) != 0) { - ave_counts /= counts.extent(0); - ave_sq_counts /= counts.extent(0); + ave_counts /= static_cast(counts.extent(0)); + ave_sq_counts /= static_cast(counts.extent(0)); ave_sq_counts -= ave_counts * ave_counts; - std::cout << "max:" << max_counts << " ave:" << ave_counts - << " var:" << ave_sq_counts << std::endl; } - return max_counts; + ostream << "max:" << max_counts << " ave:" << ave_counts + << " var:" << ave_sq_counts << std::endl; + return static_cast(max_counts); } KOKKOS_INLINE_FUNCTION - std::size_t get_max_counts() { - int max; - Kokkos::Max max_reduce(max); + auto get_max_counts() { + int max_counts; + Kokkos::Max max_reduce(max_counts); Kokkos::parallel_reduce( - "custom_velet_list::reduce_max", + "custom_verlet_list::reduce_max", Kokkos::RangePolicy(0, counts.size()), [&](const int i, int &value) { if (counts(i) > value) @@ -167,25 +142,25 @@ class CustomVerletList }, max_reduce); Kokkos::fence(); - return static_cast(max); + return max_counts; } }; template -class NeighborList< - CustomVerletList> { +class Cabana::NeighborList> { public: //! Kokkos memory space. using memory_space = MemorySpace; //! Neighbor list type. - using list_type = - CustomVerletList; + using list_type = CustomVerletList; //! Get the total number of neighbors across all particles. KOKKOS_INLINE_FUNCTION static std::size_t totalNeighbor(list_type const &list) { + std::size_t const num_p = list.counts.size(); std::size_t total_n = 0; - std::size_t num_p = list.counts.size(); for (std::size_t i = 0; i < num_p; ++i) total_n += list.counts(i); return total_n; @@ -215,6 +190,4 @@ class NeighborList< } }; -} // namespace Cabana - #endif // SHARED_MEMORY_PARALLELISM diff --git a/src/core/forces.cpp b/src/core/forces.cpp index 46b13b99dc6..52791093212 100644 --- a/src/core/forces.cpp +++ b/src/core/forces.cpp @@ -232,17 +232,18 @@ void System::System::calculate_forces() { #ifdef CALIPER CALI_MARK_BEGIN("parallel short range"); #endif + using execution_space = Kokkos::DefaultExecutionSpace; update_cabana_state(*cell_structure, verlet_criterion, get_interaction_range()); - auto &unique_particles = cell_structure->get_unique_particles(); - auto &local_force = cell_structure->get_local_force(); + auto const &unique_particles = cell_structure->get_unique_particles(); + auto const &local_force = cell_structure->get_local_force(); #ifdef ROTATION - auto &local_torque = cell_structure->get_local_torque(); + auto const &local_torque = cell_structure->get_local_torque(); #endif #ifdef NPT - auto &local_virial = cell_structure->get_local_virial(); + auto const &local_virial = cell_structure->get_local_virial(); #endif - auto const &aosoa = cell_structure->get_aosoa_data(); + auto const &aosoa = cell_structure->get_aosoa(); ForcesKernel first_neighbor_kernel( *bonded_ias, *nonbonded_ias, get_ptr(coulomb_kernel), diff --git a/src/core/forces_cabana.hpp b/src/core/forces_cabana.hpp index 6d67e76ccb1..1b3d24a4158 100644 --- a/src/core/forces_cabana.hpp +++ b/src/core/forces_cabana.hpp @@ -37,50 +37,50 @@ #endif struct ForcesKernel { - [[maybe_unused]] const BondedInteractionsMap &bonded_ias; - const InteractionsNonBonded &nonbonded_ias; - Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel; + BondedInteractionsMap const &bonded_ias; + InteractionsNonBonded const &nonbonded_ias; + Coulomb::ShortRangeForceKernel::kernel_type const *const coulomb_kernel; #if defined(LONG_RANGE_KERNELS) - Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel; + Dipoles::ShortRangeForceKernel::kernel_type const *const dipoles_kernel; Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel; - Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel; - const Thermostat::Thermostat &thermostat; + Coulomb::ShortRangeEnergyKernel::kernel_type const *const coulomb_u_kernel; + Thermostat::Thermostat const &thermostat; #endif - const BoxGeometry &box_geo; + BoxGeometry const &box_geo; #if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) - std::vector &unique_particles; + std::vector const &unique_particles; #endif - ForceType &local_force; + CellStructure::ForceType const &local_force; #ifdef ROTATION - ForceType &local_torque; + CellStructure::ForceType const &local_torque; #endif #ifdef NPT - VirialType &local_virial; + CellStructure::VirialType const &local_virial; #endif - const AoSoA_pack &aosoa; + CellStructure::AoSoA_pack const &aosoa; ForcesKernel( - [[maybe_unused]] const BondedInteractionsMap &bonded_ias_, - const InteractionsNonBonded &nonbonded_ias_, + BondedInteractionsMap const &bonded_ias_, + InteractionsNonBonded const &nonbonded_ias_, Coulomb::ShortRangeForceKernel::kernel_type const *coulomb_kernel_, #if defined(LONG_RANGE_KERNELS) Dipoles::ShortRangeForceKernel::kernel_type const *dipoles_kernel_, Coulomb::ShortRangeForceCorrectionsKernel::kernel_type const *elc_kernel_, Coulomb::ShortRangeEnergyKernel::kernel_type const *coulomb_u_kernel_, - const Thermostat::Thermostat &thermostat_, + Thermostat::Thermostat const &thermostat_, #endif - const BoxGeometry &box_geo_, + BoxGeometry const &box_geo_, #if defined(LONG_RANGE_KERNELS) or defined(EXCLUSIONS) - std::vector &unique_particles_, + std::vector const &unique_particles_, #endif - ForceType &local_force_, + CellStructure::ForceType const &local_force_, #ifdef ROTATION - ForceType &local_torque_, + CellStructure::ForceType const &local_torque_, #endif #ifdef NPT - VirialType &local_virial_, + CellStructure::VirialType const &local_virial_, #endif - const AoSoA_pack &aosoa_) + CellStructure::AoSoA_pack const &aosoa_) : bonded_ias(bonded_ias_), nonbonded_ias(nonbonded_ias_), coulomb_kernel(coulomb_kernel_), #if defined(LONG_RANGE_KERNELS) @@ -104,16 +104,16 @@ struct ForcesKernel { ESPRESSO_ATTR_ALWAYS_INLINE KOKKOS_INLINE_FUNCTION void operator()(int i, int j) const { - auto thread_id = omp_get_thread_num(); + auto const thread_id = omp_get_thread_num(); - IA_parameters const &ia_params = + auto const &ia_params = nonbonded_ias.get_ia_param(aosoa.type(i), aosoa.type(j)); ParticleForce pf{}; #ifdef NPT Utils::Vector3d virial{}; #endif - Utils::Vector3d const d = box_geo.get_mi_vector( + auto const d = box_geo.get_mi_vector( aosoa.position(i, 0), aosoa.position(i, 1), aosoa.position(i, 2), aosoa.position(j, 0), aosoa.position(j, 1), aosoa.position(j, 2)); auto const dist = d.norm(); @@ -142,7 +142,11 @@ struct ForcesKernel { } #if defined(LONG_RANGE_KERNELS) +#ifdef ELECTROSTATICS auto const q1q2 = aosoa.charge(i) * aosoa.charge(j); +#else + auto constexpr q1q2 = 0.; +#endif add_non_bonded_pair_force_with_p(p1, p2, pf, #ifdef NPT virial, @@ -162,7 +166,7 @@ struct ForcesKernel { local_torque(i, thread_id, 2) += pf.torque[2]; #endif - auto opf = calc_opposing_force(pf, d); + auto const opf = calc_opposing_force(pf, d); local_force(j, thread_id, 0) += opf.f[0]; local_force(j, thread_id, 1) += opf.f[1]; local_force(j, thread_id, 2) += opf.f[2]; diff --git a/src/core/short_range_cabana.hpp b/src/core/short_range_cabana.hpp index d54419ae094..1f1b7593fe0 100644 --- a/src/core/short_range_cabana.hpp +++ b/src/core/short_range_cabana.hpp @@ -33,31 +33,33 @@ #include #include +#include #include -inline void write_particle(Particle const &p, int const &id, - AoSoA_pack &aosoa) { - aosoa.id(id) = p.id(); - aosoa.charge(id) = p.q(); - aosoa.type(id) = p.type(); +ESPRESSO_ATTR_ALWAYS_INLINE inline void +commit_particle(Particle const &p, int const index, + CellStructure::AoSoA_pack &aosoa) { + aosoa.id(index) = p.id(); +#ifdef ELECTROSTATICS + aosoa.charge(index) = p.q(); +#endif + aosoa.type(index) = p.type(); auto const &pos = p.pos(); - for (int d = 0; d < 3; ++d) { - aosoa.position(id, d) = pos[d]; - } + aosoa.position(index, 0) = pos[0]; + aosoa.position(index, 1) = pos[1]; + aosoa.position(index, 2) = pos[2]; } ESPRESSO_ATTR_ALWAYS_INLINE inline void construct_verlet_list( - CellStructure &cell_structure, auto const &verlet_criterion, - Kokkos::View const &id_to_index, const int max_id) { - auto const &cells = - std::as_const(cell_structure).decomposition().local_cells(); - auto const distance_function = detail::MinimalImageDistance{ - std::as_const(cell_structure).decomposition().box()}; - auto &verlet_list = cell_structure.get_cabana_verlet_list(); + std::span cells, BoxGeometry const &box_geo, + CellStructure::ListType &verlet_list, auto const &verlet_criterion, + Kokkos::View const &id_to_index, int const max_id) { + + auto const distance_function = detail::MinimalImageDistance{box_geo}; // implementation detail: max_id refers to the max local particle id, // but ghost particles from other ranks may have larger particle ids; - // in addition, -1 is used as a sentinel value for particle ids + // -1 is used as a sentinel value for particle ids from other threads auto intra_kernel = [&cells, &distance_function, &verlet_criterion, &id_to_index, &verlet_list, max_id](const int i) { @@ -119,57 +121,48 @@ ESPRESSO_ATTR_ALWAYS_INLINE inline void construct_verlet_list( ESPRESSO_ATTR_ALWAYS_INLINE inline void update_cabana_state(CellStructure &cell_structure, auto const &verlet_criterion, double const pair_cutoff) { + using execution_space = Kokkos::DefaultExecutionSpace; + auto const num_threads = execution_space().concurrency(); + auto const rebuild = + cell_structure.prepare_verlet_list_cabana(num_threads, pair_cutoff); + auto const &unique_particles = cell_structure.get_unique_particles(); + auto const max_id = cell_structure.get_cached_max_local_particle_id(); + auto &aosoa = cell_structure.get_aosoa(); + + // =================================================== + // Fill particle storage + // =================================================== + Kokkos::View id_to_index( + Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); + Kokkos::deep_copy(id_to_index, -1); + + using policy_type = Kokkos::RangePolicy; + Kokkos::parallel_for( + "AoSoA write", policy_type(0, unique_particles.size()), + [&unique_particles, &aosoa, &id_to_index](int const index) { + auto const &p = *unique_particles.at(index); + commit_particle(p, index, aosoa); + id_to_index(p.id()) = index; + }); + Kokkos::fence(); - int num_threads = execution_space().concurrency(); - auto const rebuild = cell_structure.get_rebuild_cabana_verlet_list() or - (not cell_structure.use_verlet_list); - + // =================================================== + // Get Verlet pairs and fill Verlet list + // =================================================== if (rebuild) { - // If we have to rebuild, we need to count the particles - cell_structure.set_index_map(); // parallelized index_map - // Create essential variables for MD - cell_structure.rebuild_local_properties(num_threads, pair_cutoff); - } else { - // If we do not rebuild we can use the saved map - cell_structure.reset_local_properties(); - } - auto const unique_particles = cell_structure.get_unique_particles(); - auto aosoa = cell_structure.get_aosoa_data(); - auto max_id = cell_structure.get_cached_max_local_particle_id(); - - // Fill the essential variables for MD - { - // =================================================== - // Fill particle storage - // =================================================== - Kokkos::View id_to_index( - Kokkos::ViewAllocateWithoutInitializing("id_to_index"), max_id + 1); - Kokkos::deep_copy(id_to_index, -1); - - using policy_type = Kokkos::RangePolicy; - Kokkos::parallel_for( - "AoSoA write", policy_type(0, unique_particles.size()), - [&unique_particles, &aosoa, &id_to_index](const int p_id) { - write_particle(*unique_particles.at(p_id), p_id, aosoa); - id_to_index(unique_particles.at(p_id)->id()) = p_id; + cell_structure.rebuild_verlet_list_cabana( + [&](std::span cells, BoxGeometry const &box, + CellStructure::ListType &verlet_list) { + construct_verlet_list(std::move(cells), box, verlet_list, + verlet_criterion, id_to_index, max_id); }); - Kokkos::fence(); - - // =================================================== - // Get Verlet Pairs and Fill Verlet list - // =================================================== - - if (rebuild) { - construct_verlet_list(cell_structure, verlet_criterion, id_to_index, - max_id); - cell_structure.mark_rebuild_cabana_verlet_list_as_UpToDate(); - } } } void cabana_short_range(auto const &bond_kernel, auto const &forces_kernel, CellStructure &cell_structure, double pair_cutoff, double bond_cutoff) { + using execution_space = Kokkos::DefaultExecutionSpace; assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE); if (bond_cutoff >= 0.) { @@ -178,10 +171,10 @@ void cabana_short_range(auto const &bond_kernel, auto const &forces_kernel, // Cabana short range loop if (pair_cutoff > 0.) { - auto &cabana_verlet_list = cell_structure.get_cabana_verlet_list(); + auto const &verlet_list = cell_structure.get_verlet_list_cabana(); Kokkos::RangePolicy policy( 0, cell_structure.get_unique_particles().size()); - Cabana::neighbor_parallel_for(policy, forces_kernel, cabana_verlet_list, + Cabana::neighbor_parallel_for(policy, forces_kernel, verlet_list, Cabana::FirstNeighborsTag(), Cabana::SerialOpTag()); Kokkos::fence(); diff --git a/src/core/system/System.hpp b/src/core/system/System.hpp index ceb1103f33c..be53a43e7b0 100644 --- a/src/core/system/System.hpp +++ b/src/core/system/System.hpp @@ -38,7 +38,7 @@ class BoxGeometry; class LocalBox; -struct CellStructure; +class CellStructure; class Propagation; class InteractionsNonBonded; class BondedInteractionsMap; diff --git a/testsuite/python/integrator_exceptions.py b/testsuite/python/integrator_exceptions.py index f8ec6a21b8c..71559824ae1 100644 --- a/testsuite/python/integrator_exceptions.py +++ b/testsuite/python/integrator_exceptions.py @@ -179,6 +179,44 @@ def test_npt_integrator(self): self.system.lees_edwards.protocol = None self.system.integrator.run(0) + @utx.skipIfMissingFeatures(["NPT", "WCA"]) + def test_npt_integrator_negative_volume(self): + """Test for NpT with bad parameters.""" + + import tests_common + data = np.genfromtxt(tests_common.data_path("npt_lj_system.data")) + ref_box_l = np.max(data[:, 0:3]) + + system = self.system + system.part.clear() + system.cell_system.skin = 0. + + for barostat in ["Andersen", "MTK"]: + system.box_l = 3 * [ref_box_l] + system.time_step = 0.01 + if barostat == "Andersen": + piston = 0.0001 + else: + piston = 4.0 + direction = [True] * 3 + ext_pressure = 100.0 # Too large external pressure + system.part.add(pos=data[:, 0:3], v=data[:, 3:6]) + system.integrator.set_vv() + system.thermostat.set_npt(kT=1.0, gamma0=0.1, gammav=0.001, seed=42) + system.integrator.set_isotropic_npt(ext_pressure=ext_pressure, + piston=piston, + direction=direction, + barostat=barostat) + + if barostat == "Andersen": + with self.assertRaises(Exception): + system.integrator.run(10) + with self.assertRaisesRegex(Exception, "caused the volume to become negative"): + system.part.clear() + if barostat == "MTK": + # Volume cannot be negative within NPT ensemble based on MTK equation + self.assertGreater(float(np.prod(system.box_l)), 0.) + @utx.skipIfMissingFeatures("STOKESIAN_DYNAMICS") def test_stokesian_integrator(self): self.system.cell_system.skin = 0.4 diff --git a/testsuite/python/integrator_npt_stats.py b/testsuite/python/integrator_npt_stats.py index 9e3572d3d9a..e6f9c7281a7 100644 --- a/testsuite/python/integrator_npt_stats.py +++ b/testsuite/python/integrator_npt_stats.py @@ -44,6 +44,7 @@ def setUp(self): def tearDown(self): self.system.part.clear() + self.system.non_bonded_inter.reset() self.system.thermostat.turn_off() self.system.integrator.set_vv() @@ -116,91 +117,6 @@ def test_compressibility_and_pressure(self): self.assertAlmostEqual(avpV_sim, 100., delta=1.) self.assertAlmostEqual(avpV_inst, 100., delta=1.) - @utx.skipIfMissingFeatures("WCA", "P3M") - def test_pressure_compared_to_instantaneous_withP3M(self): - """Test for Npt with P3M.""" - - data = np.genfromtxt(tests_common.data_path("npt_lj_system.data")) - ref_box_l = np.max(data[:, 0:3]) - - system = self.system - system.box_l = 3 * [ref_box_l] - system.non_bonded_inter[2, 2].wca.set_params(epsilon=1., sigma=1.) - p3m = espressomd.electrostatics.P3M( - prefactor=2.0, accuracy=1e-2, mesh=3 * [14], cao=5, tune=True) - dt = 0.01 - system.time_step = dt - - p_ext = 1.0 - system.box_l = 3 * [ref_box_l] - system.part.add(pos=data[:, 0:3], type=len(data) * [2]) - system.part.all().pos = data[:, 0:3] - system.part.all().v = data[:, 3:6] - system.part.all().q = np.sign(np.arange(100) - 50 + 0.5) - self.system.integrator.set_vv() - self.system.electrostatics.solver = p3m - - if self.barostat == "Andersen": - system.thermostat.set_npt(kT=1.0, gamma0=0.2, gammav=0.01, seed=42) - system.integrator.set_isotropic_npt( - ext_pressure=p_ext, piston=0.0001) - else: - system.thermostat.set_npt( - kT=1.0, gamma0=0.5, gammav=0.001, seed=42) - system.integrator.set_isotropic_npt( - ext_pressure=p_ext, piston=4.0, barostat=self.barostat) - - steps = int(0.1 / dt) - - for _ in range(100): - system.integrator.run(steps) - p_sim = system.analysis.pressure()['total'] - p_kin = system.analysis.pressure()['kinetic'] - # virial of electrostatic force from system.analysis - p_vir = p_sim - p_kin - # virial of electrostatic force from instantaneous_pressure - p_inst_vir = system.analysis.get_instantaneous_pressure_virial() - - np.testing.assert_allclose(p_vir, p_inst_vir, atol=1e-2) - - def test_negative_volume(self): - """Test for NpT with bad parameters.""" - - data = np.genfromtxt(tests_common.data_path("npt_lj_system.data")) - ref_box_l = np.max(data[:, 0:3]) - - system = self.system - system.box_l = 3 * [ref_box_l] - dt = 0.01 - system.time_step = dt - if self.barostat == "Andersen": - piston = 0.0001 - else: - piston = 4.0 - - direction = [True] * 3 - ext_pressure = 100.0 # Too large external pressure - system.box_l = 3 * [ref_box_l] - system.part.add(pos=data[:, 0:3], type=len(data) * [2]) - system.part.all().pos = data[:, 0:3] - system.part.all().v = data[:, 3:6] - self.system.integrator.set_vv() - - system.thermostat.set_npt(kT=1.0, gamma0=0.1, gammav=0.001, seed=42) - system.integrator.set_isotropic_npt(ext_pressure=ext_pressure, - piston=piston, - direction=direction, - barostat=self.barostat) - - if self.barostat == "Andersen": - with self.assertRaises(Exception): - system.integrator.run(10) - elif self.barostat == "MTK": - with self.assertRaises(Exception): - system.integrator.run(10) - # Volume cannot be negative within NPT ensemble based on MTK equation - self.assertTrue(float(np.prod(system.box_l)) > 0.) - @utx.skipIfMissingFeatures("NPT") class IntegratorNPT_Andersen(IntegratorNPT, ut.TestCase): diff --git a/testsuite/python/npt_thermostat.py b/testsuite/python/npt_thermostat.py index aa4fc7b0219..3a1733e7e18 100644 --- a/testsuite/python/npt_thermostat.py +++ b/testsuite/python/npt_thermostat.py @@ -34,11 +34,16 @@ class NPTThermostat: def setUp(self): np.random.seed(42) + self.system.box_l = [2., 2., 2.] + self.system.time_step = 0.01 def tearDown(self): + self.system.non_bonded_inter.reset() self.system.part.clear() self.system.thermostat.turn_off() self.system.integrator.set_vv() + if espressomd.has_features("ELECTROSTATICS"): + self.system.electrostatics.clear() def test_01__rng(self): """Test for RNG consistency.""" @@ -200,6 +205,47 @@ def test_integrator_exceptions(self): system.integrator.set_isotropic_npt(ext_pressure=1., piston=1., direction=[0, 0, 0], barostat=self.barostat) + @utx.skipIfMissingFeatures(["WCA", "P3M"]) + def test_pressure_with_p3m(self): + """Test for NpT with P3M.""" + + data = np.genfromtxt(tests_common.data_path("npt_lj_system.data")) + ref_box_l = np.max(data[:, 0:3]) + p_ext = 1.0 + + system = self.system + system.box_l = 3 * [ref_box_l] + system.time_step = 0.01 + system.non_bonded_inter[2, 2].wca.set_params(epsilon=1., sigma=1.) + system.part.add(pos=data[:, 0:3], v=data[:, 3:6], type=len(data) * [2], + q=np.sign(np.arange(100) - 50 + 0.5)) + system.integrator.set_vv() + system.electrostatics.solver = espressomd.electrostatics.P3M( + prefactor=2.0, accuracy=1e-2, mesh=3 * [18], cao=5, tune=True) + + if self.barostat == "Andersen": + system.thermostat.set_npt(kT=1.0, gamma0=0.2, gammav=0.01, seed=42) + system.integrator.set_isotropic_npt( + ext_pressure=p_ext, piston=0.0001) + else: + system.thermostat.set_npt( + kT=1.0, gamma0=0.5, gammav=0.001, seed=42) + system.integrator.set_isotropic_npt( + ext_pressure=p_ext, piston=4.0, barostat=self.barostat) + + steps = int(0.1 / system.time_step) + + for _ in range(100): + system.integrator.run(steps) + p_sim = system.analysis.pressure()['total'] + p_kin = system.analysis.pressure()['kinetic'] + # virial of electrostatic force from system.analysis + p_vir = p_sim - p_kin + # virial of electrostatic force from instantaneous_pressure + p_inst_vir = system.analysis.get_instantaneous_pressure_virial() + + np.testing.assert_allclose(p_vir, p_inst_vir, rtol=1e-2, atol=1e-7) + @utx.skipIfMissingFeatures("NPT") class NPTThermostat_Andersen(NPTThermostat, ut.TestCase): From 65e699b1428e21dc0057dff077ca0ba4e16eb6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 8 Aug 2025 18:49:19 +0200 Subject: [PATCH 89/94] style --- testsuite/python/integrator_exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testsuite/python/integrator_exceptions.py b/testsuite/python/integrator_exceptions.py index 71559824ae1..b8a19fa0dec 100644 --- a/testsuite/python/integrator_exceptions.py +++ b/testsuite/python/integrator_exceptions.py @@ -195,14 +195,14 @@ def test_npt_integrator_negative_volume(self): system.box_l = 3 * [ref_box_l] system.time_step = 0.01 if barostat == "Andersen": - piston = 0.0001 + piston = 1e-4 else: piston = 4.0 direction = [True] * 3 ext_pressure = 100.0 # Too large external pressure system.part.add(pos=data[:, 0:3], v=data[:, 3:6]) system.integrator.set_vv() - system.thermostat.set_npt(kT=1.0, gamma0=0.1, gammav=0.001, seed=42) + system.thermostat.set_npt(kT=1.0, gamma0=0.1, gammav=1e-3, seed=42) system.integrator.set_isotropic_npt(ext_pressure=ext_pressure, piston=piston, direction=direction, From a86731cdb38f1bf310be85b9b42f3e8a943ccb96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 8 Aug 2025 20:12:28 +0200 Subject: [PATCH 90/94] disable test with instrumentation --- src/script_interface/code_info/CodeInfo.cpp | 7 +++++++ testsuite/python/CMakeLists.txt | 5 +++-- testsuite/python/integrator_exceptions.py | 7 +++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/script_interface/code_info/CodeInfo.cpp b/src/script_interface/code_info/CodeInfo.cpp index 868988b4d08..4f3416f1e8b 100644 --- a/src/script_interface/code_info/CodeInfo.cpp +++ b/src/script_interface/code_info/CodeInfo.cpp @@ -61,6 +61,13 @@ Variant CodeInfo::do_call_method(std::string const &name, #else // SCAFACOS return make_vector_of_variants(std::vector(0)); #endif // SCAFACOS + } + if (name == "has_fast_math") { +#if defined(__FAST_MATH__) + return true; +#else + return false; +#endif } return {}; } diff --git a/testsuite/python/CMakeLists.txt b/testsuite/python/CMakeLists.txt index 3039d83629b..517a10dc1c5 100644 --- a/testsuite/python/CMakeLists.txt +++ b/testsuite/python/CMakeLists.txt @@ -95,11 +95,12 @@ function(python_test) ${MPIEXEC} ${ESPRESSO_MPIEXEC_PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} ${TEST_NUM_PROC} ${MPIEXEC_PREFLAGS} ${ESPRESSO_MPIEXEC_TMPDIR} ${CMAKE_BINARY_DIR}/pypresso ${PYPRESSO_OPTIONS} - ${TEST_FILE_CONFIGURED} ${TEST_ARGUMENTS} ${MPIEXEC_POSTFLAGS}) + ${TEST_FILE_CONFIGURED} --verbose ${TEST_ARGUMENTS} + ${MPIEXEC_POSTFLAGS}) else() add_test(NAME ${TEST_NAME} COMMAND ${CMAKE_BINARY_DIR}/pypresso ${PYPRESSO_OPTIONS} - ${TEST_FILE_CONFIGURED} ${TEST_ARGUMENTS}) + ${TEST_FILE_CONFIGURED} --verbose ${TEST_ARGUMENTS}) endif() if(${TEST_GPU_SLOTS} GREATER 0 AND ESPRESSO_BUILD_WITH_CUDA) diff --git a/testsuite/python/integrator_exceptions.py b/testsuite/python/integrator_exceptions.py index b8a19fa0dec..143d7e2adb5 100644 --- a/testsuite/python/integrator_exceptions.py +++ b/testsuite/python/integrator_exceptions.py @@ -21,6 +21,7 @@ import espressomd.lees_edwards import espressomd.shapes import espressomd.propagation +import os import numpy as np import unittest as ut import unittest_decorators as utx @@ -179,6 +180,12 @@ def test_npt_integrator(self): self.system.lees_edwards.protocol = None self.system.integrator.run(0) + @ut.skipIf(espressomd.conde_info.call_method("has_fast_math"), + "cannot run with fast-math optimizations") + @ut.skipIf(os.environ.get("UBSAN_OPTIONS"), + "cannot run with UBSAN instrumentation") + @ut.skipIf(espressomd.has_features("FPE"), + "cannot run with FPE instrumentation") @utx.skipIfMissingFeatures(["NPT", "WCA"]) def test_npt_integrator_negative_volume(self): """Test for NpT with bad parameters.""" From 422d55951ddb7d21e8e7899db6657db6268520a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 8 Aug 2025 20:20:27 +0200 Subject: [PATCH 91/94] Fix regression --- testsuite/python/integrator_exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/python/integrator_exceptions.py b/testsuite/python/integrator_exceptions.py index 143d7e2adb5..d3f2bf176c1 100644 --- a/testsuite/python/integrator_exceptions.py +++ b/testsuite/python/integrator_exceptions.py @@ -180,7 +180,7 @@ def test_npt_integrator(self): self.system.lees_edwards.protocol = None self.system.integrator.run(0) - @ut.skipIf(espressomd.conde_info.call_method("has_fast_math"), + @ut.skipIf(espressomd.code_info._CodeInfo().call_method("has_fast_math"), "cannot run with fast-math optimizations") @ut.skipIf(os.environ.get("UBSAN_OPTIONS"), "cannot run with UBSAN instrumentation") From 0ac9c6e6eff2512e9fd11d3bc712d88533cd6f81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 8 Aug 2025 20:42:03 +0200 Subject: [PATCH 92/94] Fix error message --- testsuite/python/integrator_exceptions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/testsuite/python/integrator_exceptions.py b/testsuite/python/integrator_exceptions.py index d3f2bf176c1..0532a8b0cf4 100644 --- a/testsuite/python/integrator_exceptions.py +++ b/testsuite/python/integrator_exceptions.py @@ -216,10 +216,16 @@ def test_npt_integrator_negative_volume(self): barostat=barostat) if barostat == "Andersen": - with self.assertRaises(Exception): + exception_msg = "" + try: system.integrator.run(10) - with self.assertRaisesRegex(Exception, "caused the volume to become negative"): + except Exception as err: + exception_msg = f"{exception_msg}\n{err}" + try: system.part.clear() + except Exception as err: + exception_msg = f"{exception_msg}\n{err}" + self.assertIn("the volume to become negative", exception_msg) if barostat == "MTK": # Volume cannot be negative within NPT ensemble based on MTK equation self.assertGreater(float(np.prod(system.box_l)), 0.) From 5593a76268ca3a27c1be4917281092c9c6e8580b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 8 Aug 2025 20:44:06 +0200 Subject: [PATCH 93/94] Fix npt test --- testsuite/python/integrator_exceptions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testsuite/python/integrator_exceptions.py b/testsuite/python/integrator_exceptions.py index 0532a8b0cf4..9d99080792a 100644 --- a/testsuite/python/integrator_exceptions.py +++ b/testsuite/python/integrator_exceptions.py @@ -34,11 +34,13 @@ class Test(ut.TestCase): msg = r'while calling method integrate\(\): ERROR: ' def setUp(self): + self.system.box_l = [1., 1., 1.] self.system.part.add(pos=(0, 0, 0)) self.system.integrator.set_vv() self.system.periodicity = 3 * [True] def tearDown(self): + self.system.box_l = [1., 1., 1.] self.system.thermostat.turn_off() self.system.part.clear() self.system.constraints.clear() From 910c06d4a98a1b3ad2e7c91929acf0b8cb5805ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-No=C3=ABl=20Grad?= Date: Fri, 8 Aug 2025 20:48:24 +0200 Subject: [PATCH 94/94] fixup --- testsuite/python/integrator_exceptions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testsuite/python/integrator_exceptions.py b/testsuite/python/integrator_exceptions.py index 9d99080792a..bc3c573f6e1 100644 --- a/testsuite/python/integrator_exceptions.py +++ b/testsuite/python/integrator_exceptions.py @@ -40,7 +40,6 @@ def setUp(self): self.system.periodicity = 3 * [True] def tearDown(self): - self.system.box_l = [1., 1., 1.] self.system.thermostat.turn_off() self.system.part.clear() self.system.constraints.clear() @@ -232,6 +231,9 @@ def test_npt_integrator_negative_volume(self): # Volume cannot be negative within NPT ensemble based on MTK equation self.assertGreater(float(np.prod(system.box_l)), 0.) + system.part.clear() + system.box_l = [1., 1., 1.] + @utx.skipIfMissingFeatures("STOKESIAN_DYNAMICS") def test_stokesian_integrator(self): self.system.cell_system.skin = 0.4