From c2fe36e0b54c758499a47a31130d11849a23f1a8 Mon Sep 17 00:00:00 2001 From: itischler Date: Thu, 17 Dec 2020 18:51:53 +0100 Subject: [PATCH 1/7] Added MRT relaxation rates interface, which makes them accesable via the python interface. If they are not set directly they will be set via viscosity and magic number --- .../grid_based_algorithms/lb_interface.cpp | 34 +++++++++- .../grid_based_algorithms/lb_interface.hpp | 10 +++ .../lb_walberla_instance.cpp | 43 ++++++++++-- .../lb_walberla_instance.hpp | 17 ++++- src/python/espressomd/lb.pxd | 23 ++++++- src/python/espressomd/lb.pyx | 66 +++++++++++++++++-- src/walberla_bridge/LBWalberlaBase.hpp | 21 +++++- .../LBWalberlaD3Q19FluctuatingMRT.hpp | 53 +++++++++++++-- src/walberla_bridge/LBWalberlaD3Q19MRT.hpp | 46 +++++++++++-- src/walberla_bridge/LBWalberlaImpl.hpp | 2 +- src/walberla_bridge/lb_walberla_init.cpp | 32 +++++++-- src/walberla_bridge/lb_walberla_init.hpp | 9 ++- src/walberla_bridge/walberla_utils.hpp | 2 + testsuite/python/lb_walberla.py | 19 ++++++ 14 files changed, 338 insertions(+), 39 deletions(-) diff --git a/src/core/grid_based_algorithms/lb_interface.cpp b/src/core/grid_based_algorithms/lb_interface.cpp index 4ec39eadd3a..92be831e549 100644 --- a/src/core/grid_based_algorithms/lb_interface.cpp +++ b/src/core/grid_based_algorithms/lb_interface.cpp @@ -158,8 +158,38 @@ double lb_lbfluid_get_viscosity() { } double lb_lbfluid_get_bulk_viscosity() { - if (lattice_switch == ActiveLB::WALBERLA) - throw std::runtime_error("Getting bulk viscosity not implemented."); +#ifdef LB_WALBERLA + if (lattice_switch == ActiveLB::WALBERLA) { + return lb_walberla()->get_bulk_viscosity(); + } +#endif + throw NoLBActive(); +} + +double lb_lbfluid_get_shear_viscosity() { +#ifdef LB_WALBERLA + if (lattice_switch == ActiveLB::WALBERLA) { + return lb_walberla()->get_shear_viscosity(); + } +#endif + throw NoLBActive(); +} + +double lb_lbfluid_get_gamma_odd() { +#ifdef LB_WALBERLA + if (lattice_switch == ActiveLB::WALBERLA) { + return lb_walberla()->get_gamma_odd(); + } +#endif + throw NoLBActive(); +} + +double lb_lbfluid_get_gamma_even() { +#ifdef LB_WALBERLA + if (lattice_switch == ActiveLB::WALBERLA) { + return lb_walberla()->get_gamma_even(); + } +#endif throw NoLBActive(); } diff --git a/src/core/grid_based_algorithms/lb_interface.hpp b/src/core/grid_based_algorithms/lb_interface.hpp index 67268f29301..5df4b6d6275 100644 --- a/src/core/grid_based_algorithms/lb_interface.hpp +++ b/src/core/grid_based_algorithms/lb_interface.hpp @@ -125,11 +125,21 @@ double lb_lbfluid_get_agrid(); */ double lb_lbfluid_get_gamma_even(); +/** + * @brief Get the global LB relaxation parameter for odd modes. + */ +double lb_lbfluid_get_gamma_odd(); + /** * @brief Get the global LB bulk viscosity. */ double lb_lbfluid_get_bulk_viscosity(); +/** + * @brief Get the global LB shear viscosity. + */ +double lb_lbfluid_get_shear_viscosity(); + /** * @brief Get the global LB viscosity. */ diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.cpp b/src/core/grid_based_algorithms/lb_walberla_instance.cpp index d09ab96f6c6..24f6d270ad0 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.cpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.cpp @@ -55,14 +55,14 @@ LBWalberlaParams *lb_walberla_params() { return lb_walberla_params_instance; } -void init_lb_walberla(double viscosity, double density, double agrid, +void init_lb_walberla_magic_number(double viscosity, double magic_number, double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, double kT, unsigned int seed) { // Exceptions need to be converted to runtime errors so they can be // handled from Python in a parallel simulation try { - lb_walberla_instance = new_lb_walberla(viscosity, density, agrid, tau, + lb_walberla_instance = new_lb_walberla(viscosity, magic_number, density, agrid, tau, box_dimensions, node_grid, kT, seed); lb_walberla_params_instance = new LBWalberlaParams{agrid, tau}; } catch (const std::exception &e) { @@ -71,7 +71,25 @@ void init_lb_walberla(double viscosity, double density, double agrid, lb_walberla_params_instance = nullptr; } } -REGISTER_CALLBACK(init_lb_walberla) +REGISTER_CALLBACK(init_lb_walberla_magic_number) + +void init_lb_walberla_relaxation_rates(LBRelaxationRates relaxation_rates, double density, double agrid, + double tau, const Utils::Vector3d &box_dimensions, + const Utils::Vector3i &node_grid, double kT, + unsigned int seed) { + // Exceptions need to be converted to runtime errors so they can be + // handled from Python in a parallel simulation + try { + lb_walberla_instance = new_lb_walberla(relaxation_rates, density, agrid, tau, + box_dimensions, node_grid, kT, seed); + lb_walberla_params_instance = new LBWalberlaParams{agrid, tau}; + } catch (const std::exception &e) { + runtimeErrorMsg() << "Error during Walberla initialization: " << e.what(); + lb_walberla_instance = nullptr; + lb_walberla_params_instance = nullptr; + } +} +REGISTER_CALLBACK(init_lb_walberla_relaxation_rates) void destruct_lb_walberla() { delete lb_walberla_instance; @@ -81,10 +99,21 @@ void destruct_lb_walberla() { } REGISTER_CALLBACK(destruct_lb_walberla) -void mpi_init_lb_walberla(double viscosity, double density, double agrid, - double tau, double kT, unsigned int seed) { - Communication::mpiCallbacks().call_all(init_lb_walberla, viscosity, density, - agrid, tau, box_geo.length(), +void mpi_init_lb_walberla(double viscosity, double magic_number, double density, + double agrid, double tau, double kT, unsigned int seed){ + Communication::mpiCallbacks().call_all(init_lb_walberla_magic_number, viscosity, magic_number, + density, agrid, tau, box_geo.length(), + node_grid, kT, seed); + if (lb_walberla_instance) { + lb_lbfluid_set_lattice_switch(ActiveLB::WALBERLA); + lb_lbfluid_sanity_checks(); + } +} + +void mpi_init_lb_walberla(LBRelaxationRates relaxation_rates, double density, double agrid, + double tau, double kT, unsigned int seed){ + Communication::mpiCallbacks().call_all(init_lb_walberla_relaxation_rates, relaxation_rates, + density, agrid, tau, box_geo.length(), node_grid, kT, seed); if (lb_walberla_instance) { lb_lbfluid_set_lattice_switch(ActiveLB::WALBERLA); diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.hpp b/src/core/grid_based_algorithms/lb_walberla_instance.hpp index 4e020d14d07..ce0c1926967 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.hpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.hpp @@ -47,12 +47,25 @@ LBWalberlaParams *lb_walberla_params(); * WALBERLA * * @param viscosity Fluid viscosity + * @param magic_number Fluid magic number * @param density Fluid density * @param agrid Size of one lb cell * @param tau LB time step */ -void mpi_init_lb_walberla(double viscosity, double density, double agrid, - double tau, double kT, unsigned int seed); +void mpi_init_lb_walberla(double viscosity, double magic_number, double density, + double agrid, double tau, double kT, unsigned int seed); + + +/** @brief Create the LBWalberla instance and sets the lattice switch to + * WALBERLA + * + * @param relaxation_rates Fluid relaxation rates + * @param density Fluid density + * @param agrid Size of one lb cell + * @param tau LB time step + */ +void mpi_init_lb_walberla(LBRelaxationRates relaxation_rates, double density, + double agrid, double tau, double kT, unsigned int seed); /** @brief Destruct the LBWalberla instance and set lattice switch to NONE */ void mpi_destruct_lb_walberla(); diff --git a/src/python/espressomd/lb.pxd b/src/python/espressomd/lb.pxd index fb206b0eeb4..df66eaa60fe 100644 --- a/src/python/espressomd/lb.pxd +++ b/src/python/espressomd/lb.pxd @@ -38,6 +38,14 @@ cdef class HydrodynamicInteraction(Actor): # IF LB_WALBERLA: + cdef extern from "LBWalberlaBase.hpp": + cdef cppclass LBRelaxationRates: + double omega_bulk; + double omega_shear; + double omega_odd; + double omega_even; + + cdef extern from "grid_based_algorithms/lb_interface.hpp": cdef enum OutputVTK: @@ -64,6 +72,9 @@ cdef extern from "grid_based_algorithms/lb_interface.hpp": void lb_lbfluid_set_ext_force_density(const Vector3d forcedensity) except + const Vector3d lb_lbfluid_get_ext_force_density() except + double lb_lbfluid_get_bulk_viscosity() except + + double lb_lbfluid_get_shear_viscosity() except + + double lb_lbfluid_get_gamma_odd() except + + double lb_lbfluid_get_gamma_even() except + void lb_lbfluid_sanity_checks() except + void lb_lbfluid_save_checkpoint(string filename, bool binary) except + void lb_lbfluid_load_checkpoint(string filename, bool binary) except + @@ -111,7 +122,8 @@ cdef extern from "grid_based_algorithms/lb_interpolation.hpp" namespace "Interpo IF LB_WALBERLA: cdef extern from "grid_based_algorithms/lb_walberla_instance.hpp": - void mpi_init_lb_walberla(double viscosity, double density, double agrid, double tau, double kT, unsigned int seed) except + + void mpi_init_lb_walberla(double viscosity, double magic_number, double density, double agrid, double tau, double kT, unsigned int seed) except + + void mpi_init_lb_walberla(LBRelaxationRates relaxation_rates, double density, double agrid, double tau, double kT, unsigned int seed) except + void mpi_destruct_lb_walberla() except + @@ -142,6 +154,15 @@ cdef inline double python_lbfluid_get_viscosity(p_agrid, p_tau) except +: cdef inline double python_lbfluid_get_bulk_viscosity(p_agrid, p_tau) except +: return lb_lbfluid_get_bulk_viscosity() / p_tau * (p_agrid * p_agrid) +cdef inline double python_lbfluid_get_shear_viscosity(p_agrid, p_tau) except +: + return lb_lbfluid_get_shear_viscosity() / p_tau * (p_agrid * p_agrid) + +cdef inline double python_lbfluid_get_gamma_odd(p_agrid, p_tau) except +: + return lb_lbfluid_get_gamma_odd() / p_tau * (p_agrid * p_agrid) + +cdef inline double python_lbfluid_get_gamma_even(p_agrid, p_tau) except +: + return lb_lbfluid_get_gamma_even() / p_tau * (p_agrid * p_agrid) + cdef inline double python_lbfluid_get_gamma() except +: return lb_lbcoupling_get_gamma() diff --git a/src/python/espressomd/lb.pyx b/src/python/espressomd/lb.pyx index 7d4cb3598e1..3dea74eccbf 100644 --- a/src/python/espressomd/lb.pyx +++ b/src/python/espressomd/lb.pyx @@ -183,8 +183,12 @@ cdef class HydrodynamicInteraction(Actor): Fluid density. visc : :obj:`float` Fluid kinematic viscosity. + magic_number : :obj:`float` + Magic number to define the relaxation time from viscosity. bulk_visc : :obj:`float`, optional Fluid bulk viscosity. + shear_visc : :obj:`float`, optional + Fluid shear viscosity. gamma_odd : :obj:`int`, optional Relaxation parameter :math:`\\gamma_{\\textrm{odd}}` for kinetic modes. gamma_even : :obj:`int`, optional @@ -231,10 +235,10 @@ cdef class HydrodynamicInteraction(Actor): raise ValueError("tau has to be a positive double") def valid_keys(self): - return "agrid", "dens", "ext_force_density", "visc", "tau", "bulk_visc", "gamma_odd", "gamma_even", "kT", "seed" + return "agrid", "dens", "ext_force_density", "visc", "magic_number", "tau", "bulk_visc", "shear_visc", "gamma_odd", "gamma_even", "kT", "seed" def required_keys(self): - return ["dens", "agrid", "visc", "tau"] + return ["dens", "agrid", "tau"] def default_params(self): return {"agrid": -1.0, @@ -365,6 +369,22 @@ cdef class HydrodynamicInteraction(Actor): def __get__(self): return python_lbfluid_get_viscosity(self.agrid, self.tau) + property bulk_viscosity: + def __get__(self): + return python_lbfluid_get_bulk_viscosity(self.agrid, self.tau) + + property shear_viscosity: + def __get__(self): + return python_lbfluid_get_shear_viscosity(self.agrid, self.tau) + + property gamma_odd: + def __get__(self): + return python_lbfluid_get_gamma_odd(self.agrid, self.tau) + + property gamma_even: + def __get__(self): + return python_lbfluid_get_gamma_even(self.agrid, self.tau) + property tau: def __get__(self): return lb_lbfluid_get_tau() @@ -399,6 +419,10 @@ IF LB_WALBERLA: "ext_force_density": [0.0, 0.0, 0.0], "visc": -1.0, "bulk_visc": -1.0, + "shear_visc": -1.0, + "gamma_odd": -1.0, + "gamma_even": -1.0, + "magic_number": 3.0/16.0, "tau": -1.0, "kT": 0.0, "seed": 0} @@ -407,17 +431,47 @@ IF LB_WALBERLA: raise Exception("This may not be called") def _activate_method(self): + cdef LBRelaxationRates relaxation_rates + self.validate_params() # unit conversion lb_visc = self._params["visc"] * \ self._params['tau'] / self._params['agrid']**2 + lb_magic_number = self._params["magic_number"] + + lb_bulk_visc = self._params["bulk_visc"] * \ + self._params['tau'] / self._params['agrid']**2 + lb_shear_visc = self._params["shear_visc"] * \ + self._params['tau'] / self._params['agrid']**2 + lb_gamma_odd = self._params["gamma_odd"] * \ + self._params['tau'] / self._params['agrid']**2 + lb_gamma_even = self._params["gamma_even"] * \ + self._params['tau'] / self._params['agrid']**2 + lb_dens = self._params['dens'] * self._params['agrid']**3 lb_kT = self._params['kT'] * \ - self._params['tau']**2 / self._params['agrid']**2 - mpi_init_lb_walberla( - lb_visc, lb_dens, self._params["agrid"], self._params["tau"], - lb_kT, self._params['seed']) + self._params['tau']**2 / self._params['agrid']**2 + + default_params = self.default_params() + + if ((not lb_bulk_visc == default_params["bulk_visc"]) and + (not lb_shear_visc == default_params["shear_visc"]) and + (not lb_gamma_odd == default_params["gamma_odd"]) and + (not lb_gamma_even == default_params["gamma_even"])): + + relaxation_rates.omega_bulk = lb_bulk_visc + relaxation_rates.omega_shear = lb_shear_visc + relaxation_rates.omega_odd = lb_gamma_odd + relaxation_rates.omega_even = lb_gamma_even + mpi_init_lb_walberla( + relaxation_rates, lb_dens, self._params["agrid"], + self._params["tau"], lb_kT, self._params['seed']) + else: + mpi_init_lb_walberla( + lb_visc, lb_magic_number, lb_dens, + self._params["agrid"], self._params["tau"], + lb_kT, self._params['seed']) utils.handle_errors("LB fluid activation") self.ext_force_density = self._params["ext_force_density"] diff --git a/src/walberla_bridge/LBWalberlaBase.hpp b/src/walberla_bridge/LBWalberlaBase.hpp index f42d1933389..816e2a49d24 100644 --- a/src/walberla_bridge/LBWalberlaBase.hpp +++ b/src/walberla_bridge/LBWalberlaBase.hpp @@ -105,8 +105,12 @@ class LBWalberlaBase { virtual Utils::Vector3d get_external_force() const = 0; // Global parameters - virtual void set_viscosity(double viscosity) = 0; + virtual void set_viscosity(double viscosity, double magic_number) = 0; virtual double get_viscosity() const = 0; + virtual double get_bulk_viscosity() const = 0; + virtual double get_shear_viscosity() const = 0; + virtual double get_gamma_odd() const = 0; + virtual double get_gamma_even() const = 0; virtual double get_kT() const = 0; // Grid, domain, halo @@ -159,4 +163,19 @@ enum class OutputVTK : unsigned { pressure_tensor = 1u << 2u, }; +struct LBRelaxationRates { + double omega_bulk; + double omega_shear; + double omega_odd; + double omega_even; + template + void serialize(Archive &ar, const unsigned int version) { + ar &omega_bulk; + ar &omega_shear; + ar &omega_odd; + ar &omega_even; + } +}; + + #endif // LB_WALBERLA_H diff --git a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp index dab4a85b0d3..691cc6a1786 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp @@ -8,9 +8,9 @@ class LBWalberlaD3Q19FluctuatingMRT using LatticeModel = lbm::FluctuatingMRTLatticeModel; public: - void construct_lattice_model(double viscosity, double kT, unsigned int seed) { + void construct_lattice_model(double viscosity, double magic_number, + double kT, unsigned int seed) { const real_t omega = 2 / (6 * real_c(viscosity) + 1); - const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); m_lattice_model = std::make_shared( @@ -21,10 +21,19 @@ class LBWalberlaD3Q19FluctuatingMRT omega, // shear 1, seed)); }; - void set_viscosity(double viscosity) override { + void construct_lattice_model(LBRelaxationRates relaxation_rates, + double kT, unsigned int seed) { + m_lattice_model = std::make_shared( + LatticeModel(m_last_applied_force_field_id, real_c(kT), + relaxation_rates.omega_bulk, + relaxation_rates.omega_even, + relaxation_rates.omega_odd, + relaxation_rates.omega_shear, + 1, seed)); + }; + void set_viscosity(double viscosity, double magic_number) override { LatticeModel *lm = dynamic_cast(m_lattice_model.get()); const real_t omega = 2 / (6 * real_c(viscosity) + 1); - const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); lm->omega_shear_ = omega; @@ -37,16 +46,46 @@ class LBWalberlaD3Q19FluctuatingMRT LatticeModel *lm = dynamic_cast(m_lattice_model.get()); return (2 - lm->omega_shear_) / (6 * lm->omega_shear_); }; - LBWalberlaD3Q19FluctuatingMRT(double viscosity, double density, double agrid, + double get_bulk_viscosity() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_bulk_; + }; + double get_shear_viscosity() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_shear_; + }; + double get_gamma_odd() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_odd_; + }; + double get_gamma_even() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_even_; + }; + LBWalberlaD3Q19FluctuatingMRT(double viscosity, double magic_number, + double density, double agrid, + double tau, + const Utils::Vector3d &box_dimensions, + const Utils::Vector3i &node_grid, + int n_ghost_layers, double kT, + unsigned int seed) + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, + n_ghost_layers) { + m_kT = kT; + construct_lattice_model(viscosity, magic_number, kT, seed); + setup_with_valid_lattice_model(density); + }; + LBWalberlaD3Q19FluctuatingMRT(LBRelaxationRates relaxation_rates, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, int n_ghost_layers, double kT, unsigned int seed) - : LBWalberlaImpl(viscosity, agrid, tau, box_dimensions, node_grid, + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, n_ghost_layers) { m_kT = kT; - construct_lattice_model(viscosity, kT, seed); + construct_lattice_model(relaxation_rates, kT, seed); setup_with_valid_lattice_model(density); }; void integrate() override { diff --git a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp index 15c009e99cb..5470d39af43 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp @@ -6,9 +6,8 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { public: using LatticeModel = lbm::MRTLatticeModel; - void construct_lattice_model(double viscosity) { + void construct_lattice_model(double viscosity, double magic_number) { const real_t omega = 2 / (6 * real_c(viscosity) + 1); - const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); m_lattice_model = std::make_shared( @@ -18,10 +17,17 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { omega_2, // odd omega)); // shear }; - void set_viscosity(double viscosity) override { + void construct_lattice_model(LBRelaxationRates relaxation_rates) { + m_lattice_model = std::make_shared( + LatticeModel(m_last_applied_force_field_id, + relaxation_rates.omega_bulk, + relaxation_rates.omega_even, + relaxation_rates.omega_odd, + relaxation_rates.omega_shear)); + }; + void set_viscosity(double viscosity, double magic_number) override { LatticeModel *lm = dynamic_cast(m_lattice_model.get()); const real_t omega = 2 / (6 * real_c(viscosity) + 1); - const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); lm->omega_shear_ = omega; @@ -34,12 +40,38 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { LatticeModel *lm = dynamic_cast(m_lattice_model.get()); return (2 - lm->omega_shear_) / (6 * lm->omega_shear_); }; - LBWalberlaD3Q19MRT(double viscosity, double density, double agrid, double tau, + double get_bulk_viscosity() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_bulk_; + }; + double get_shear_viscosity() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_shear_; + }; + double get_gamma_odd() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_odd_; + }; + double get_gamma_even() const override { + LatticeModel *lm = dynamic_cast(m_lattice_model.get()); + return lm->omega_even_; + }; + LBWalberlaD3Q19MRT(double viscosity, double magic_number, double density, + double agrid, double tau, + const Utils::Vector3d &box_dimensions, + const Utils::Vector3i &node_grid, int n_ghost_layers) + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, + n_ghost_layers) { + construct_lattice_model(viscosity, magic_number); + setup_with_valid_lattice_model(density); + }; + LBWalberlaD3Q19MRT(LBRelaxationRates relaxation_rates, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, int n_ghost_layers) - : LBWalberlaImpl(viscosity, agrid, tau, box_dimensions, node_grid, + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, n_ghost_layers) { - construct_lattice_model(viscosity); + construct_lattice_model(relaxation_rates); setup_with_valid_lattice_model(density); }; }; diff --git a/src/walberla_bridge/LBWalberlaImpl.hpp b/src/walberla_bridge/LBWalberlaImpl.hpp index 26b15817064..6e7816b6b33 100644 --- a/src/walberla_bridge/LBWalberlaImpl.hpp +++ b/src/walberla_bridge/LBWalberlaImpl.hpp @@ -231,7 +231,7 @@ template class LBWalberlaImpl : public LBWalberlaBase { }; public: - LBWalberlaImpl(double viscosity, double agrid, double tau, + LBWalberlaImpl(double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, int n_ghost_layers) { m_n_ghost_layers = n_ghost_layers; diff --git a/src/walberla_bridge/lb_walberla_init.cpp b/src/walberla_bridge/lb_walberla_init.cpp index dcb502cf010..4da7d6a6ad9 100644 --- a/src/walberla_bridge/lb_walberla_init.cpp +++ b/src/walberla_bridge/lb_walberla_init.cpp @@ -32,7 +32,8 @@ void walberla_mpi_init() { walberla::mpi::Environment(argc, argv); } -LBWalberlaBase *new_lb_walberla(double viscosity, double density, double agrid, +LBWalberlaBase *new_lb_walberla(double viscosity, double magic_number, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, double kT, @@ -42,12 +43,35 @@ LBWalberlaBase *new_lb_walberla(double viscosity, double density, double agrid, if (kT == 0.) { // un-thermalized LB lb_walberla_instance = new walberla::LBWalberlaD3Q19MRT(walberla::LBWalberlaD3Q19MRT{ - viscosity, density, agrid, tau, box_dimensions, node_grid, 1}); + viscosity, magic_number, density, agrid, tau, box_dimensions, node_grid, 1}); } else { // thermalized LB lb_walberla_instance = new walberla::LBWalberlaD3Q19FluctuatingMRT( - walberla::LBWalberlaD3Q19FluctuatingMRT{viscosity, density, agrid, tau, - box_dimensions, node_grid, 1, + walberla::LBWalberlaD3Q19FluctuatingMRT{viscosity, magic_number, density, + agrid, tau, box_dimensions, node_grid, 1, kT, seed}); } return lb_walberla_instance; } + +LBWalberlaBase *new_lb_walberla(LBRelaxationRates relaxation_rates, + double density, double agrid, + double tau, + const Utils::Vector3d &box_dimensions, + const Utils::Vector3i &node_grid, double kT, + unsigned int seed) { + + LBWalberlaBase *lb_walberla_instance; + if (kT == 0.) { // un-thermalized LB + lb_walberla_instance = + new walberla::LBWalberlaD3Q19MRT(walberla::LBWalberlaD3Q19MRT{ + relaxation_rates, density, agrid, tau, box_dimensions, node_grid, 1}); + } else { // thermalized LB + lb_walberla_instance = new walberla::LBWalberlaD3Q19FluctuatingMRT( + walberla::LBWalberlaD3Q19FluctuatingMRT{relaxation_rates, density, + agrid, tau, box_dimensions, node_grid, 1, + kT, seed}); + } + return lb_walberla_instance; +} + + diff --git a/src/walberla_bridge/lb_walberla_init.hpp b/src/walberla_bridge/lb_walberla_init.hpp index 0a30d2af5b5..52465e8379f 100644 --- a/src/walberla_bridge/lb_walberla_init.hpp +++ b/src/walberla_bridge/lb_walberla_init.hpp @@ -25,10 +25,17 @@ void walberla_mpi_init(); -LBWalberlaBase *new_lb_walberla(double viscosity, double density, double agrid, +LBWalberlaBase *new_lb_walberla(double viscosity, double magic_number, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, double kT, unsigned int seed); +LBWalberlaBase *new_lb_walberla(LBRelaxationRates relaxation_rates, double density, + double agrid, double tau, + const Utils::Vector3d &box_dimensions, + const Utils::Vector3i &node_grid, double kT, + unsigned int seed); + #endif diff --git a/src/walberla_bridge/walberla_utils.hpp b/src/walberla_bridge/walberla_utils.hpp index fc55fa39a9a..f412d96fc67 100644 --- a/src/walberla_bridge/walberla_utils.hpp +++ b/src/walberla_bridge/walberla_utils.hpp @@ -25,6 +25,8 @@ #include #include +#include + #include #include diff --git a/testsuite/python/lb_walberla.py b/testsuite/python/lb_walberla.py index 50de3e1fcf8..8211bcd5a68 100644 --- a/testsuite/python/lb_walberla.py +++ b/testsuite/python/lb_walberla.py @@ -34,6 +34,25 @@ def test(self): s.time_step = 0.2 s.cell_system.skin = 0. dens_init = 1.3 + bulk_visc = 0.4 + shear_visc = 0.3 + gamma_odd = 0.1 + gamma_even = 0.2 + lbf = LBFluidWalberla( + agrid=.6, + dens=dens_init, + bulk_visc=bulk_visc, + shear_visc=shear_visc, + gamma_odd=gamma_odd, + gamma_even=gamma_even, + tau=s.time_step) + s.actors.add(lbf) + np.testing.assert_almost_equal(lbf.bulk_viscosity, bulk_visc) + np.testing.assert_almost_equal(lbf.shear_viscosity, shear_visc) + np.testing.assert_almost_equal(lbf.gamma_odd, gamma_odd) + np.testing.assert_almost_equal(lbf.gamma_even, gamma_even) + s.actors.remove(lbf) + lbf = LBFluidWalberla( agrid=.6, dens=dens_init, From d6f6a0bdad6077ed74cc789a39b66b4100623379 Mon Sep 17 00:00:00 2001 From: itischler Date: Thu, 17 Dec 2020 18:59:51 +0100 Subject: [PATCH 2/7] formatting --- .../lb_walberla_instance.cpp | 48 +++++++++++-------- .../lb_walberla_instance.hpp | 7 +-- src/walberla_bridge/LBWalberlaBase.hpp | 1 - .../LBWalberlaD3Q19FluctuatingMRT.hpp | 31 +++++------- src/walberla_bridge/LBWalberlaD3Q19MRT.hpp | 16 +++---- src/walberla_bridge/lb_walberla_init.cpp | 30 +++++------- src/walberla_bridge/lb_walberla_init.hpp | 7 ++- 7 files changed, 65 insertions(+), 75 deletions(-) diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.cpp b/src/core/grid_based_algorithms/lb_walberla_instance.cpp index 24f6d270ad0..27459188f66 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.cpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.cpp @@ -55,15 +55,17 @@ LBWalberlaParams *lb_walberla_params() { return lb_walberla_params_instance; } -void init_lb_walberla_magic_number(double viscosity, double magic_number, double density, double agrid, - double tau, const Utils::Vector3d &box_dimensions, - const Utils::Vector3i &node_grid, double kT, - unsigned int seed) { +void init_lb_walberla_magic_number(double viscosity, double magic_number, + double density, double agrid, double tau, + const Utils::Vector3d &box_dimensions, + const Utils::Vector3i &node_grid, double kT, + unsigned int seed) { // Exceptions need to be converted to runtime errors so they can be // handled from Python in a parallel simulation try { - lb_walberla_instance = new_lb_walberla(viscosity, magic_number, density, agrid, tau, - box_dimensions, node_grid, kT, seed); + lb_walberla_instance = + new_lb_walberla(viscosity, magic_number, density, agrid, tau, + box_dimensions, node_grid, kT, seed); lb_walberla_params_instance = new LBWalberlaParams{agrid, tau}; } catch (const std::exception &e) { runtimeErrorMsg() << "Error during Walberla initialization: " << e.what(); @@ -73,15 +75,17 @@ void init_lb_walberla_magic_number(double viscosity, double magic_number, double } REGISTER_CALLBACK(init_lb_walberla_magic_number) -void init_lb_walberla_relaxation_rates(LBRelaxationRates relaxation_rates, double density, double agrid, - double tau, const Utils::Vector3d &box_dimensions, - const Utils::Vector3i &node_grid, double kT, - unsigned int seed) { +void init_lb_walberla_relaxation_rates(LBRelaxationRates relaxation_rates, + double density, double agrid, double tau, + const Utils::Vector3d &box_dimensions, + const Utils::Vector3i &node_grid, + double kT, unsigned int seed) { // Exceptions need to be converted to runtime errors so they can be // handled from Python in a parallel simulation try { - lb_walberla_instance = new_lb_walberla(relaxation_rates, density, agrid, tau, - box_dimensions, node_grid, kT, seed); + lb_walberla_instance = + new_lb_walberla(relaxation_rates, density, agrid, tau, box_dimensions, + node_grid, kT, seed); lb_walberla_params_instance = new LBWalberlaParams{agrid, tau}; } catch (const std::exception &e) { runtimeErrorMsg() << "Error during Walberla initialization: " << e.what(); @@ -100,21 +104,23 @@ void destruct_lb_walberla() { REGISTER_CALLBACK(destruct_lb_walberla) void mpi_init_lb_walberla(double viscosity, double magic_number, double density, - double agrid, double tau, double kT, unsigned int seed){ - Communication::mpiCallbacks().call_all(init_lb_walberla_magic_number, viscosity, magic_number, - density, agrid, tau, box_geo.length(), - node_grid, kT, seed); + double agrid, double tau, double kT, + unsigned int seed) { + Communication::mpiCallbacks().call_all( + init_lb_walberla_magic_number, viscosity, magic_number, density, agrid, + tau, box_geo.length(), node_grid, kT, seed); if (lb_walberla_instance) { lb_lbfluid_set_lattice_switch(ActiveLB::WALBERLA); lb_lbfluid_sanity_checks(); } } -void mpi_init_lb_walberla(LBRelaxationRates relaxation_rates, double density, double agrid, - double tau, double kT, unsigned int seed){ - Communication::mpiCallbacks().call_all(init_lb_walberla_relaxation_rates, relaxation_rates, - density, agrid, tau, box_geo.length(), - node_grid, kT, seed); +void mpi_init_lb_walberla(LBRelaxationRates relaxation_rates, double density, + double agrid, double tau, double kT, + unsigned int seed) { + Communication::mpiCallbacks().call_all(init_lb_walberla_relaxation_rates, + relaxation_rates, density, agrid, tau, + box_geo.length(), node_grid, kT, seed); if (lb_walberla_instance) { lb_lbfluid_set_lattice_switch(ActiveLB::WALBERLA); lb_lbfluid_sanity_checks(); diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.hpp b/src/core/grid_based_algorithms/lb_walberla_instance.hpp index ce0c1926967..4aaaf9b7b96 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.hpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.hpp @@ -53,8 +53,8 @@ LBWalberlaParams *lb_walberla_params(); * @param tau LB time step */ void mpi_init_lb_walberla(double viscosity, double magic_number, double density, - double agrid, double tau, double kT, unsigned int seed); - + double agrid, double tau, double kT, + unsigned int seed); /** @brief Create the LBWalberla instance and sets the lattice switch to * WALBERLA @@ -65,7 +65,8 @@ void mpi_init_lb_walberla(double viscosity, double magic_number, double density, * @param tau LB time step */ void mpi_init_lb_walberla(LBRelaxationRates relaxation_rates, double density, - double agrid, double tau, double kT, unsigned int seed); + double agrid, double tau, double kT, + unsigned int seed); /** @brief Destruct the LBWalberla instance and set lattice switch to NONE */ void mpi_destruct_lb_walberla(); diff --git a/src/walberla_bridge/LBWalberlaBase.hpp b/src/walberla_bridge/LBWalberlaBase.hpp index 816e2a49d24..5ddca58456a 100644 --- a/src/walberla_bridge/LBWalberlaBase.hpp +++ b/src/walberla_bridge/LBWalberlaBase.hpp @@ -177,5 +177,4 @@ struct LBRelaxationRates { } }; - #endif // LB_WALBERLA_H diff --git a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp index 691cc6a1786..cdc32c522fb 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp @@ -8,8 +8,8 @@ class LBWalberlaD3Q19FluctuatingMRT using LatticeModel = lbm::FluctuatingMRTLatticeModel; public: - void construct_lattice_model(double viscosity, double magic_number, - double kT, unsigned int seed) { + void construct_lattice_model(double viscosity, double magic_number, double kT, + unsigned int seed) { const real_t omega = 2 / (6 * real_c(viscosity) + 1); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); @@ -21,15 +21,12 @@ class LBWalberlaD3Q19FluctuatingMRT omega, // shear 1, seed)); }; - void construct_lattice_model(LBRelaxationRates relaxation_rates, - double kT, unsigned int seed) { - m_lattice_model = std::make_shared( - LatticeModel(m_last_applied_force_field_id, real_c(kT), - relaxation_rates.omega_bulk, - relaxation_rates.omega_even, - relaxation_rates.omega_odd, - relaxation_rates.omega_shear, - 1, seed)); + void construct_lattice_model(LBRelaxationRates relaxation_rates, double kT, + unsigned int seed) { + m_lattice_model = std::make_shared(LatticeModel( + m_last_applied_force_field_id, real_c(kT), relaxation_rates.omega_bulk, + relaxation_rates.omega_even, relaxation_rates.omega_odd, + relaxation_rates.omega_shear, 1, seed)); }; void set_viscosity(double viscosity, double magic_number) override { LatticeModel *lm = dynamic_cast(m_lattice_model.get()); @@ -63,27 +60,23 @@ class LBWalberlaD3Q19FluctuatingMRT return lm->omega_even_; }; LBWalberlaD3Q19FluctuatingMRT(double viscosity, double magic_number, - double density, double agrid, - double tau, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, int n_ghost_layers, double kT, unsigned int seed) - : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, - n_ghost_layers) { + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, n_ghost_layers) { m_kT = kT; construct_lattice_model(viscosity, magic_number, kT, seed); setup_with_valid_lattice_model(density); }; LBWalberlaD3Q19FluctuatingMRT(LBRelaxationRates relaxation_rates, - double density, double agrid, - double tau, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, int n_ghost_layers, double kT, unsigned int seed) - : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, - n_ghost_layers) { + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, n_ghost_layers) { m_kT = kT; construct_lattice_model(relaxation_rates, kT, seed); setup_with_valid_lattice_model(density); diff --git a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp index 5470d39af43..a71b25b1aa8 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp @@ -19,10 +19,8 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { }; void construct_lattice_model(LBRelaxationRates relaxation_rates) { m_lattice_model = std::make_shared( - LatticeModel(m_last_applied_force_field_id, - relaxation_rates.omega_bulk, - relaxation_rates.omega_even, - relaxation_rates.omega_odd, + LatticeModel(m_last_applied_force_field_id, relaxation_rates.omega_bulk, + relaxation_rates.omega_even, relaxation_rates.omega_odd, relaxation_rates.omega_shear)); }; void set_viscosity(double viscosity, double magic_number) override { @@ -60,17 +58,15 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, int n_ghost_layers) - : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, - n_ghost_layers) { + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, n_ghost_layers) { construct_lattice_model(viscosity, magic_number); setup_with_valid_lattice_model(density); }; - LBWalberlaD3Q19MRT(LBRelaxationRates relaxation_rates, - double density, double agrid, double tau, + LBWalberlaD3Q19MRT(LBRelaxationRates relaxation_rates, double density, + double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, int n_ghost_layers) - : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, - n_ghost_layers) { + : LBWalberlaImpl(agrid, tau, box_dimensions, node_grid, n_ghost_layers) { construct_lattice_model(relaxation_rates); setup_with_valid_lattice_model(density); }; diff --git a/src/walberla_bridge/lb_walberla_init.cpp b/src/walberla_bridge/lb_walberla_init.cpp index 4da7d6a6ad9..205b36c7b34 100644 --- a/src/walberla_bridge/lb_walberla_init.cpp +++ b/src/walberla_bridge/lb_walberla_init.cpp @@ -33,45 +33,41 @@ void walberla_mpi_init() { } LBWalberlaBase *new_lb_walberla(double viscosity, double magic_number, - double density, double agrid, - double tau, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, double kT, unsigned int seed) { LBWalberlaBase *lb_walberla_instance; if (kT == 0.) { // un-thermalized LB - lb_walberla_instance = - new walberla::LBWalberlaD3Q19MRT(walberla::LBWalberlaD3Q19MRT{ - viscosity, magic_number, density, agrid, tau, box_dimensions, node_grid, 1}); + lb_walberla_instance = new walberla::LBWalberlaD3Q19MRT( + walberla::LBWalberlaD3Q19MRT{viscosity, magic_number, density, agrid, + tau, box_dimensions, node_grid, 1}); } else { // thermalized LB lb_walberla_instance = new walberla::LBWalberlaD3Q19FluctuatingMRT( - walberla::LBWalberlaD3Q19FluctuatingMRT{viscosity, magic_number, density, - agrid, tau, box_dimensions, node_grid, 1, - kT, seed}); + walberla::LBWalberlaD3Q19FluctuatingMRT{ + viscosity, magic_number, density, agrid, tau, box_dimensions, + node_grid, 1, kT, seed}); } return lb_walberla_instance; } LBWalberlaBase *new_lb_walberla(LBRelaxationRates relaxation_rates, - double density, double agrid, - double tau, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, double kT, unsigned int seed) { LBWalberlaBase *lb_walberla_instance; if (kT == 0.) { // un-thermalized LB - lb_walberla_instance = - new walberla::LBWalberlaD3Q19MRT(walberla::LBWalberlaD3Q19MRT{ - relaxation_rates, density, agrid, tau, box_dimensions, node_grid, 1}); + lb_walberla_instance = new walberla::LBWalberlaD3Q19MRT( + walberla::LBWalberlaD3Q19MRT{relaxation_rates, density, agrid, tau, + box_dimensions, node_grid, 1}); } else { // thermalized LB lb_walberla_instance = new walberla::LBWalberlaD3Q19FluctuatingMRT( walberla::LBWalberlaD3Q19FluctuatingMRT{relaxation_rates, density, - agrid, tau, box_dimensions, node_grid, 1, - kT, seed}); + agrid, tau, box_dimensions, + node_grid, 1, kT, seed}); } return lb_walberla_instance; } - - diff --git a/src/walberla_bridge/lb_walberla_init.hpp b/src/walberla_bridge/lb_walberla_init.hpp index 52465e8379f..90c481eccad 100644 --- a/src/walberla_bridge/lb_walberla_init.hpp +++ b/src/walberla_bridge/lb_walberla_init.hpp @@ -26,14 +26,13 @@ void walberla_mpi_init(); LBWalberlaBase *new_lb_walberla(double viscosity, double magic_number, - double density, double agrid, - double tau, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, double kT, unsigned int seed); -LBWalberlaBase *new_lb_walberla(LBRelaxationRates relaxation_rates, double density, - double agrid, double tau, +LBWalberlaBase *new_lb_walberla(LBRelaxationRates relaxation_rates, + double density, double agrid, double tau, const Utils::Vector3d &box_dimensions, const Utils::Vector3i &node_grid, double kT, unsigned int seed); From 68f405f3aff63d29006b06f6fac4ba878843de08 Mon Sep 17 00:00:00 2001 From: itischler Date: Fri, 28 May 2021 10:25:25 +0200 Subject: [PATCH 3/7] merge fix --- .../grid_based_algorithms/lb_interface.cpp | 34 +----------- .../grid_based_algorithms/lb_interface.hpp | 7 +-- .../lb_walberla_instance.cpp | 24 +-------- .../lb_walberla_instance.hpp | 3 +- src/python/espressomd/lb.pxd | 27 +--------- src/python/espressomd/lb.pyx | 52 +------------------ src/walberla_bridge/LBWalberlaBase.hpp | 22 +------- .../LBWalberlaD3Q19FluctuatingMRT.hpp | 9 ++-- src/walberla_bridge/LBWalberlaD3Q19MRT.hpp | 4 +- testsuite/python/lb_walberla.py | 18 ------- 10 files changed, 18 insertions(+), 182 deletions(-) diff --git a/src/core/grid_based_algorithms/lb_interface.cpp b/src/core/grid_based_algorithms/lb_interface.cpp index fc1e51ff744..096e8cf14c2 100644 --- a/src/core/grid_based_algorithms/lb_interface.cpp +++ b/src/core/grid_based_algorithms/lb_interface.cpp @@ -158,38 +158,8 @@ double lb_lbfluid_get_viscosity() { } double lb_lbfluid_get_bulk_viscosity() { -#ifdef LB_WALBERLA - if (lattice_switch == ActiveLB::WALBERLA) { - return lb_walberla()->get_bulk_viscosity(); - } -#endif - throw NoLBActive(); -} - -double lb_lbfluid_get_shear_viscosity() { -#ifdef LB_WALBERLA - if (lattice_switch == ActiveLB::WALBERLA) { - return lb_walberla()->get_shear_viscosity(); - } -#endif - throw NoLBActive(); -} - -double lb_lbfluid_get_gamma_odd() { -#ifdef LB_WALBERLA - if (lattice_switch == ActiveLB::WALBERLA) { - return lb_walberla()->get_gamma_odd(); - } -#endif - throw NoLBActive(); -} - -double lb_lbfluid_get_gamma_even() { -#ifdef LB_WALBERLA - if (lattice_switch == ActiveLB::WALBERLA) { - return lb_walberla()->get_gamma_even(); - } -#endif + if (lattice_switch == ActiveLB::WALBERLA) + throw std::runtime_error("Getting bulk viscosity not implemented."); throw NoLBActive(); } diff --git a/src/core/grid_based_algorithms/lb_interface.hpp b/src/core/grid_based_algorithms/lb_interface.hpp index af9333bd081..e5fc9cdf231 100644 --- a/src/core/grid_based_algorithms/lb_interface.hpp +++ b/src/core/grid_based_algorithms/lb_interface.hpp @@ -125,11 +125,6 @@ double lb_lbfluid_get_agrid(); */ double lb_lbfluid_get_bulk_viscosity(); -/** - * @brief Get the global LB shear viscosity. - */ -double lb_lbfluid_get_shear_viscosity(); - /** * @brief Get the global LB viscosity. */ @@ -251,4 +246,4 @@ void lb_lbfluid_add_force_at_pos(const Utils::Vector3d &pos, */ double lb_lbfluid_get_interpolated_density(const Utils::Vector3d &pos); -#endif +#endif \ No newline at end of file diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.cpp b/src/core/grid_based_algorithms/lb_walberla_instance.cpp index f04b3c5659a..a3693d59f2e 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.cpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.cpp @@ -71,27 +71,7 @@ void init_lb_walberla(double viscosity, double density, double agrid, lb_walberla_params_instance = nullptr; } } -REGISTER_CALLBACK(init_lb_walberla_magic_number) - -void init_lb_walberla_relaxation_rates(LBRelaxationRates relaxation_rates, - double density, double agrid, double tau, - const Utils::Vector3d &box_dimensions, - const Utils::Vector3i &node_grid, - double kT, unsigned int seed) { - // Exceptions need to be converted to runtime errors so they can be - // handled from Python in a parallel simulation - try { - lb_walberla_instance = - new_lb_walberla(relaxation_rates, density, agrid, tau, box_dimensions, - node_grid, kT, seed); - lb_walberla_params_instance = new LBWalberlaParams{agrid, tau}; - } catch (const std::exception &e) { - runtimeErrorMsg() << "Error during Walberla initialization: " << e.what(); - lb_walberla_instance = nullptr; - lb_walberla_params_instance = nullptr; - } -} -REGISTER_CALLBACK(init_lb_walberla_relaxation_rates) +REGISTER_CALLBACK(init_lb_walberla) void destruct_lb_walberla() { delete lb_walberla_instance; @@ -129,4 +109,4 @@ void mpi_destruct_lb_walberla() { lb_lbfluid_set_lattice_switch(ActiveLB::NONE); Communication::mpiCallbacks().call_all(destruct_lb_walberla); } -#endif +#endif \ No newline at end of file diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.hpp b/src/core/grid_based_algorithms/lb_walberla_instance.hpp index 06c9dc2a28d..0b85c67c1ab 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.hpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.hpp @@ -50,7 +50,6 @@ void init_lb_walberla(double viscosity, double density, double agrid, * WALBERLA * * @param viscosity Fluid viscosity - * @param magic_number Fluid magic number * @param density Fluid density * @param agrid Size of one lb cell * @param tau LB time step @@ -64,4 +63,4 @@ void mpi_destruct_lb_walberla(); #endif // LB_WALBERLA -#endif +#endif \ No newline at end of file diff --git a/src/python/espressomd/lb.pxd b/src/python/espressomd/lb.pxd index 8ab040ee673..d994a92e402 100644 --- a/src/python/espressomd/lb.pxd +++ b/src/python/espressomd/lb.pxd @@ -41,14 +41,6 @@ cdef class LBFluidRoutines: ############################################## IF LB_WALBERLA: - cdef extern from "LBWalberlaBase.hpp": - cdef cppclass LBRelaxationRates: - double omega_bulk; - double omega_shear; - double omega_odd; - double omega_even; - - cdef extern from "grid_based_algorithms/lb_interface.hpp": cdef enum OutputVTK: @@ -75,9 +67,6 @@ cdef extern from "grid_based_algorithms/lb_interface.hpp": void lb_lbfluid_set_ext_force_density(const Vector3d forcedensity) except + const Vector3d lb_lbfluid_get_ext_force_density() except + double lb_lbfluid_get_bulk_viscosity() except + - double lb_lbfluid_get_shear_viscosity() except + - double lb_lbfluid_get_gamma_odd() except + - double lb_lbfluid_get_gamma_even() except + void lb_lbfluid_sanity_checks() except + void lb_lbfluid_save_checkpoint(string filename, bool binary) except + void lb_lbfluid_load_checkpoint(string filename, bool binary) except + @@ -126,12 +115,7 @@ cdef extern from "grid_based_algorithms/lb_interpolation.hpp" namespace "Interpo IF LB_WALBERLA: cdef extern from "grid_based_algorithms/lb_walberla_instance.hpp": -<<<<<<< HEAD - void mpi_init_lb_walberla(double viscosity, double magic_number, double density, double agrid, double tau, double kT, unsigned int seed) except + - void mpi_init_lb_walberla(LBRelaxationRates relaxation_rates, double density, double agrid, double tau, double kT, unsigned int seed) except + -======= void mpi_init_lb_walberla(double viscosity, double density, double agrid, double tau, Vector3d box_size, double kT, unsigned int seed) except + ->>>>>>> walberla void mpi_destruct_lb_walberla() except + @@ -162,15 +146,6 @@ cdef inline double python_lbfluid_get_viscosity(p_agrid, p_tau) except +: cdef inline double python_lbfluid_get_bulk_viscosity(p_agrid, p_tau) except +: return lb_lbfluid_get_bulk_viscosity() / p_tau * (p_agrid * p_agrid) -cdef inline double python_lbfluid_get_shear_viscosity(p_agrid, p_tau) except +: - return lb_lbfluid_get_shear_viscosity() / p_tau * (p_agrid * p_agrid) - -cdef inline double python_lbfluid_get_gamma_odd(p_agrid, p_tau) except +: - return lb_lbfluid_get_gamma_odd() / p_tau * (p_agrid * p_agrid) - -cdef inline double python_lbfluid_get_gamma_even(p_agrid, p_tau) except +: - return lb_lbfluid_get_gamma_even() / p_tau * (p_agrid * p_agrid) - cdef inline double python_lbfluid_get_gamma() except +: return lb_lbcoupling_get_gamma() @@ -234,4 +209,4 @@ cdef inline python_lbfluid_set_gamma(p_gamma) except +: else: c_gamma = p_gamma # call c-function - lb_lbcoupling_set_gamma(c_gamma) + lb_lbcoupling_set_gamma(c_gamma) \ No newline at end of file diff --git a/src/python/espressomd/lb.pyx b/src/python/espressomd/lb.pyx index 8957f1426de..aa8ce7bddb5 100644 --- a/src/python/espressomd/lb.pyx +++ b/src/python/espressomd/lb.pyx @@ -118,7 +118,6 @@ IF LB_WALBERLA: class VTKOutputAutomatic(VTKOutput): """ Automatic VTK callback. Can be disabled at any time and re-enabled later. - Please note that the internal VTK counter is no longer incremented when the callback is disabled, which means the number of LB steps between two frames will not always be an integer multiple of delta_N. @@ -173,7 +172,6 @@ def _construct(cls, params): cdef class HydrodynamicInteraction(Actor): """ Base class for LB implementations. - Parameters ---------- agrid : :obj:`float` @@ -185,12 +183,8 @@ cdef class HydrodynamicInteraction(Actor): Fluid density. visc : :obj:`float` Fluid kinematic viscosity. - magic_number : :obj:`float` - Magic number to define the relaxation time from viscosity. bulk_visc : :obj:`float`, optional Fluid bulk viscosity. - shear_visc : :obj:`float`, optional - Fluid shear viscosity. gamma_odd : :obj:`int`, optional Relaxation parameter :math:`\\gamma_{\\textrm{odd}}` for kinetic modes. gamma_even : :obj:`int`, optional @@ -283,17 +277,14 @@ cdef class HydrodynamicInteraction(Actor): def get_interpolated_velocity(self, pos): """Get LB fluid velocity at specified position. - Parameters ---------- pos : (3,) array_like of :obj:`float` The position at which velocity is requested. - Returns ------- v : (3,) array_like :obj:`float` The LB fluid velocity at ``pos``. - """ cdef Vector3d p @@ -304,14 +295,12 @@ cdef class HydrodynamicInteraction(Actor): def add_force_at_pos(self, pos, force): """Adds a force to the fluid at given position - Parameters ---------- pos : (3,) array_like of :obj:`float` The position at which the force will be added. force : (3,) array_like of :obj:`float` The force vector which will be distributed at the position. - """ cdef Vector3d p cdef Vector3d f @@ -378,22 +367,6 @@ cdef class HydrodynamicInteraction(Actor): def __get__(self): return python_lbfluid_get_viscosity(self.agrid, self.tau) - property bulk_viscosity: - def __get__(self): - return python_lbfluid_get_bulk_viscosity(self.agrid, self.tau) - - property shear_viscosity: - def __get__(self): - return python_lbfluid_get_shear_viscosity(self.agrid, self.tau) - - property gamma_odd: - def __get__(self): - return python_lbfluid_get_gamma_odd(self.agrid, self.tau) - - property gamma_even: - def __get__(self): - return python_lbfluid_get_gamma_even(self.agrid, self.tau) - property tau: def __get__(self): return lb_lbfluid_get_tau() @@ -416,7 +389,6 @@ IF LB_WALBERLA: """ Initialize the lattice-Boltzmann method for hydrodynamic flow using waLBerla. See :class:`HydrodynamicInteraction` for the list of parameters. - """ def _set_params_in_es_core(self): @@ -428,10 +400,6 @@ IF LB_WALBERLA: "ext_force_density": [0.0, 0.0, 0.0], "visc": -1.0, "bulk_visc": -1.0, - "shear_visc": -1.0, - "gamma_odd": -1.0, - "gamma_even": -1.0, - "magic_number": 3.0/16.0, "tau": -1.0, "kT": 0.0, "seed": 0} @@ -440,24 +408,11 @@ IF LB_WALBERLA: raise Exception("This may not be called") def _activate_method(self): - cdef LBRelaxationRates relaxation_rates - self.validate_params() # unit conversion lb_visc = self._params["visc"] * \ self._params['tau'] / self._params['agrid']**2 - lb_magic_number = self._params["magic_number"] - - lb_bulk_visc = self._params["bulk_visc"] * \ - self._params['tau'] / self._params['agrid']**2 - lb_shear_visc = self._params["shear_visc"] * \ - self._params['tau'] / self._params['agrid']**2 - lb_gamma_odd = self._params["gamma_odd"] * \ - self._params['tau'] / self._params['agrid']**2 - lb_gamma_even = self._params["gamma_even"] * \ - self._params['tau'] / self._params['agrid']**2 - lb_dens = self._params['dens'] * self._params['agrid']**3 lb_kT = self._params['kT'] * \ self._params['tau']**2 / self._params['agrid']**2 @@ -476,10 +431,8 @@ IF LB_WALBERLA: base_folder='vtk_out', prefix='simulation_step'): """ Create a VTK observable. - Files are written to ``//_*.vtu``. Summary is written to ``/.pvd``. - Parameters ---------- identifier : :obj:`str` @@ -645,14 +598,12 @@ class LBSlice: def _add_lb_slice_properties(): """ Automatically add all of LBFluidRoutines's properties to LBSlice. - """ def set_attribute(lb_slice, value, attribute): """ Setter function that sets attribute on every member of lb_slice. If values contains only one element, all members are set to it. - """ indices = [lb_slice.x_indices, lb_slice.y_indices, lb_slice.z_indices] @@ -680,7 +631,6 @@ def _add_lb_slice_properties(): """ Getter function that copies attribute from every member of lb_slice into an array (if possible). - """ indices = [lb_slice.x_indices, lb_slice.y_indices, lb_slice.z_indices] @@ -705,4 +655,4 @@ def _add_lb_slice_properties(): setattr(LBSlice, attribute_name, new_property) -_add_lb_slice_properties() +_add_lb_slice_properties() \ No newline at end of file diff --git a/src/walberla_bridge/LBWalberlaBase.hpp b/src/walberla_bridge/LBWalberlaBase.hpp index b3a8678930c..18873743e64 100644 --- a/src/walberla_bridge/LBWalberlaBase.hpp +++ b/src/walberla_bridge/LBWalberlaBase.hpp @@ -116,12 +116,8 @@ class LBWalberlaBase { virtual Utils::Vector3d get_external_force() const = 0; // Global parameters - virtual void set_viscosity(double viscosity, double magic_number) = 0; + virtual void set_viscosity(double viscosity) = 0; virtual double get_viscosity() const = 0; - virtual double get_bulk_viscosity() const = 0; - virtual double get_shear_viscosity() const = 0; - virtual double get_gamma_odd() const = 0; - virtual double get_gamma_even() const = 0; virtual double get_kT() const = 0; // Grid, domain, halo @@ -176,18 +172,4 @@ enum class OutputVTK : unsigned { pressure_tensor = 1u << 2u, }; -struct LBRelaxationRates { - double omega_bulk; - double omega_shear; - double omega_odd; - double omega_even; - template - void serialize(Archive &ar, const unsigned int version) { - ar &omega_bulk; - ar &omega_shear; - ar &omega_odd; - ar &omega_even; - } -}; - -#endif // LB_WALBERLA_H +#endif // LB_WALBERLA_H \ No newline at end of file diff --git a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp index 6d4e89b69c4..95c7eb34afd 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp @@ -14,9 +14,9 @@ class LBWalberlaD3Q19FluctuatingMRT : public LBWalberlaImpl { using LatticeModel = LatticeModelName; public: - void construct_lattice_model(double viscosity, double magic_number, double kT, - unsigned int seed) { + void construct_lattice_model(double viscosity, double kT, unsigned int seed) { const real_t omega = 2 / (6 * real_c(viscosity) + 1); + const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); m_lattice_model = std::make_shared( @@ -30,6 +30,7 @@ class LBWalberlaD3Q19FluctuatingMRT : public LBWalberlaImpl { void set_viscosity(double viscosity) override { auto *lm = dynamic_cast(m_lattice_model.get()); const real_t omega = 2 / (6 * real_c(viscosity) + 1); + const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); lm->omega_shear_ = omega; @@ -49,7 +50,7 @@ class LBWalberlaD3Q19FluctuatingMRT : public LBWalberlaImpl { unsigned int seed) : LBWalberlaImpl(viscosity, grid_dimensions, node_grid, n_ghost_layers) { m_kT = kT; - construct_lattice_model(relaxation_rates, kT, seed); + construct_lattice_model(viscosity, kT, seed); setup_with_valid_lattice_model(density); }; void integrate() override { @@ -65,4 +66,4 @@ class LBWalberlaD3Q19FluctuatingMRT : public LBWalberlaImpl { }; } // namespace walberla -#undef LatticeModelName +#undef LatticeModelName \ No newline at end of file diff --git a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp index 2f2ef9386b5..eabb913dbfe 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp @@ -14,6 +14,7 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { public: void construct_lattice_model(double viscosity) { const real_t omega = 2 / (6 * real_c(viscosity) + 1); + const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); m_lattice_model = std::make_shared( @@ -26,6 +27,7 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { void set_viscosity(double viscosity) override { auto *lm = dynamic_cast(m_lattice_model.get()); const real_t omega = 2 / (6 * real_c(viscosity) + 1); + const real_t magic_number = real_c(3.) / real_c(16.); const real_t omega_2 = (4 - 2 * omega) / (4 * magic_number * omega + 2 - omega); lm->omega_shear_ = omega; @@ -49,4 +51,4 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { } // namespace walberla -#undef LatticeModelName +#undef LatticeModelName \ No newline at end of file diff --git a/testsuite/python/lb_walberla.py b/testsuite/python/lb_walberla.py index 8211bcd5a68..e89ea7629eb 100644 --- a/testsuite/python/lb_walberla.py +++ b/testsuite/python/lb_walberla.py @@ -34,24 +34,6 @@ def test(self): s.time_step = 0.2 s.cell_system.skin = 0. dens_init = 1.3 - bulk_visc = 0.4 - shear_visc = 0.3 - gamma_odd = 0.1 - gamma_even = 0.2 - lbf = LBFluidWalberla( - agrid=.6, - dens=dens_init, - bulk_visc=bulk_visc, - shear_visc=shear_visc, - gamma_odd=gamma_odd, - gamma_even=gamma_even, - tau=s.time_step) - s.actors.add(lbf) - np.testing.assert_almost_equal(lbf.bulk_viscosity, bulk_visc) - np.testing.assert_almost_equal(lbf.shear_viscosity, shear_visc) - np.testing.assert_almost_equal(lbf.gamma_odd, gamma_odd) - np.testing.assert_almost_equal(lbf.gamma_even, gamma_even) - s.actors.remove(lbf) lbf = LBFluidWalberla( agrid=.6, From f206778890a46fdefc86f6a27a78df4cd5a9391d Mon Sep 17 00:00:00 2001 From: itischler Date: Fri, 28 May 2021 10:25:55 +0200 Subject: [PATCH 4/7] newly generated kernels with pressure tensor update --- .../generated_kernels/Dynamic_UBB.cpp | 183 +- .../FluctuatingMRTLatticeModel.cpp | 2810 ++++----- .../FluctuatingMRTLatticeModel.h | 95 +- .../FluctuatingMRTLatticeModelAvx.cpp | 2811 ++++----- .../FluctuatingMRTLatticeModelAvx.h | 95 +- .../generated_kernels/MRTLatticeModel.cpp | 2410 ++++---- .../generated_kernels/MRTLatticeModel.h | 95 +- .../generated_kernels/MRTLatticeModelAvx.cpp | 5050 +++++------------ .../generated_kernels/MRTLatticeModelAvx.h | 95 +- .../generated_kernels/PushPackInfo.cpp | 3631 +++++------- 10 files changed, 6661 insertions(+), 10614 deletions(-) diff --git a/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp b/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp index 955dc005041..64e652aff6d 100644 --- a/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp +++ b/src/walberla_bridge/generated_kernels/Dynamic_UBB.cpp @@ -19,9 +19,11 @@ #include -#include "Dynamic_UBB.h" #include "core/DataTypes.h" #include "core/Macros.h" +#include "Dynamic_UBB.h" + + #define FUNC_PREFIX @@ -43,102 +45,38 @@ namespace lbm { #endif namespace internal_dynamic_ubb_boundary_Dynamic_UBB { -static FUNC_PREFIX void dynamic_ubb_boundary_Dynamic_UBB( - uint8_t *RESTRICT const _data_indexVector, double *RESTRICT _data_pdfs, - int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t indexVectorSize) { - - const int32_t f_in_inv_dir_idx[] = {0, 2, 1, 4, 3, 6, 5, 10, 9, 8, - 7, 16, 15, 18, 17, 12, 11, 14, 13}; - const int32_t f_out_offsets_x[] = {0, 0, 0, -1, 1, 0, 0, -1, 1, -1, - 1, 0, 0, -1, 1, 0, 0, -1, 1}; - const int32_t f_out_offsets_y[] = {0, 1, -1, 0, 0, 0, 0, 1, 1, -1, - -1, 1, -1, 0, 0, 1, -1, 0, 0}; - const int32_t f_out_offsets_z[] = {0, 0, 0, 0, 0, 1, -1, 0, 0, 0, - 0, 1, 1, 1, 1, -1, -1, -1, -1}; - - const double weights[] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; - - const int32_t neighbour_offset_x[] = {0, 0, 0, -1, 1, 0, 0, -1, 1, -1, - 1, 0, 0, -1, 1, 0, 0, -1, 1}; - const int32_t neighbour_offset_y[] = {0, 1, -1, 0, 0, 0, 0, 1, 1, -1, - -1, 1, -1, 0, 0, 1, -1, 0, 0}; - const int32_t neighbour_offset_z[] = {0, 0, 0, 0, 0, 1, -1, 0, 0, 0, - 0, 1, 1, 1, 1, -1, -1, -1, -1}; - - for (int64_t ctr_0 = 0; ctr_0 < indexVectorSize; ctr_0 += 1) { - const int32_t x = *((int32_t *)(&_data_indexVector[40 * ctr_0])); - const int32_t y = *((int32_t *)(&_data_indexVector[40 * ctr_0 + 4])); - const int32_t z = *((int32_t *)(&_data_indexVector[40 * ctr_0 + 8])); - const int32_t dir = *((int32_t *)(&_data_indexVector[40 * ctr_0 + 12])); - const double vel0Term = - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_1 + _stride_pdfs_2 * z + 8 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + 4 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + _stride_pdfs_2 + 14 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z - _stride_pdfs_2 + 18 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 + _stride_pdfs_1 * y - - _stride_pdfs_1 + _stride_pdfs_2 * z + 10 * _stride_pdfs_3]; - const double vel1Term = - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_1 + - _stride_pdfs_2 * z + _stride_pdfs_2 + 11 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_1 + - _stride_pdfs_2 * z + _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_1 + - _stride_pdfs_2 * z - _stride_pdfs_2 + 15 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_1 + _stride_pdfs_2 * z + 7 * _stride_pdfs_3]; - const double vel2Term = - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + _stride_pdfs_2 + 5 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y - _stride_pdfs_1 + - _stride_pdfs_2 * z + _stride_pdfs_2 + 12 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + _stride_pdfs_2 + 13 * _stride_pdfs_3]; - const double rho = - vel0Term + vel1Term + vel2Term + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + - _stride_pdfs_2 * z - _stride_pdfs_2 + 6 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + - _stride_pdfs_2 * z] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y - _stride_pdfs_1 + - _stride_pdfs_2 * z + 2 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y - _stride_pdfs_1 + - _stride_pdfs_2 * z - _stride_pdfs_2 + 16 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z + 3 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y + - _stride_pdfs_2 * z - _stride_pdfs_2 + 17 * _stride_pdfs_3] + - _data_pdfs[_stride_pdfs_0 * x - _stride_pdfs_0 + _stride_pdfs_1 * y - - _stride_pdfs_1 + _stride_pdfs_2 * z + 9 * _stride_pdfs_3]; - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_1 * y + _stride_pdfs_2 * z + - _stride_pdfs_3 * f_in_inv_dir_idx[dir]] = - -rho * - (6.0 * *((double *)(&_data_indexVector[40 * ctr_0 + 16])) * - neighbour_offset_x[dir] + - 6.0 * *((double *)(&_data_indexVector[40 * ctr_0 + 24])) * - neighbour_offset_y[dir] + - 6.0 * *((double *)(&_data_indexVector[40 * ctr_0 + 32])) * - neighbour_offset_z[dir]) * - weights[dir] + - _data_pdfs[_stride_pdfs_0 * x + _stride_pdfs_0 * f_out_offsets_x[dir] + - _stride_pdfs_1 * y + _stride_pdfs_1 * f_out_offsets_y[dir] + - _stride_pdfs_2 * z + _stride_pdfs_2 * f_out_offsets_z[dir] + - _stride_pdfs_3 * dir]; - } +static FUNC_PREFIX void dynamic_ubb_boundary_Dynamic_UBB(uint8_t * RESTRICT const _data_indexVector, double * RESTRICT _data_pdfs, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t indexVectorSize) +{ + + const int32_t f_in_inv_dir_idx [] = { 0,2,1,4,3,6,5,10,9,8,7,16,15,18,17,12,11,14,13 }; + const int32_t f_out_offsets_x [] = { 0,0,0,-1,1,0,0,-1,1,-1,1,0,0,-1,1,0,0,-1,1 }; + const int32_t f_out_offsets_y [] = { 0,1,-1,0,0,0,0,1,1,-1,-1,1,-1,0,0,1,-1,0,0 }; + const int32_t f_out_offsets_z [] = { 0,0,0,0,0,1,-1,0,0,0,0,1,1,1,1,-1,-1,-1,-1 }; + + + const double weights [] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; + + + + const int32_t neighbour_offset_x [] = { 0,0,0,-1,1,0,0,-1,1,-1,1,0,0,-1,1,0,0,-1,1 }; + const int32_t neighbour_offset_y [] = { 0,1,-1,0,0,0,0,1,1,-1,-1,1,-1,0,0,1,-1,0,0 }; + const int32_t neighbour_offset_z [] = { 0,0,0,0,0,1,-1,0,0,0,0,1,1,1,1,-1,-1,-1,-1 }; + + for (int64_t ctr_0 = 0; ctr_0 < indexVectorSize; ctr_0 += 1) + { + const int32_t x = *((int32_t *)(& _data_indexVector[40*ctr_0])); + const int32_t y = *((int32_t *)(& _data_indexVector[40*ctr_0 + 4])); + const int32_t z = *((int32_t *)(& _data_indexVector[40*ctr_0 + 8])); + const int32_t dir = *((int32_t *)(& _data_indexVector[40*ctr_0 + 12])); + const double vel0Term = _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + 8*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + 4*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_2 + 14*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z - _stride_pdfs_2 + 18*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0 + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + 10*_stride_pdfs_3]; + const double vel1Term = _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + _stride_pdfs_2 + 11*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + _stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z - _stride_pdfs_2 + 15*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_1 + _stride_pdfs_2*z + 7*_stride_pdfs_3]; + const double vel2Term = _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_2 + 5*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + _stride_pdfs_2 + 12*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_2 + 13*_stride_pdfs_3]; + const double rho = vel0Term + vel1Term + vel2Term + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z - _stride_pdfs_2 + 6*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + 2*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z - _stride_pdfs_2 + 16*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z + 3*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y + _stride_pdfs_2*z - _stride_pdfs_2 + 17*_stride_pdfs_3] + _data_pdfs[_stride_pdfs_0*x - _stride_pdfs_0 + _stride_pdfs_1*y - _stride_pdfs_1 + _stride_pdfs_2*z + 9*_stride_pdfs_3]; + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_1*y + _stride_pdfs_2*z + _stride_pdfs_3*f_in_inv_dir_idx[dir]] = -rho*(6.0**((double *)(& _data_indexVector[40*ctr_0 + 16]))*neighbour_offset_x[dir] + 6.0**((double *)(& _data_indexVector[40*ctr_0 + 24]))*neighbour_offset_y[dir] + 6.0**((double *)(& _data_indexVector[40*ctr_0 + 32]))*neighbour_offset_z[dir])*weights[dir] + _data_pdfs[_stride_pdfs_0*x + _stride_pdfs_0*f_out_offsets_x[dir] + _stride_pdfs_1*y + _stride_pdfs_1*f_out_offsets_y[dir] + _stride_pdfs_2*z + _stride_pdfs_2*f_out_offsets_z[dir] + _stride_pdfs_3*dir]; + } +} } -} // namespace internal_dynamic_ubb_boundary_Dynamic_UBB + #ifdef __GNUC__ #pragma GCC diagnostic pop @@ -148,34 +86,47 @@ static FUNC_PREFIX void dynamic_ubb_boundary_Dynamic_UBB( #pragma pop #endif -void Dynamic_UBB::run_impl(IBlock *block, IndexVectors::Type type) { - auto *indexVectors = block->getData(indexVectorID); - int64_t indexVectorSize = int64_c(indexVectors->indexVector(type).size()); - if (indexVectorSize == 0) - return; - auto pointer = indexVectors->pointerCpu(type); +void Dynamic_UBB::run_impl(IBlock * block, IndexVectors::Type type) +{ + auto * indexVectors = block->getData(indexVectorID); + int64_t indexVectorSize = int64_c( indexVectors->indexVector(type).size() ); + if( indexVectorSize == 0) + return; - uint8_t *_data_indexVector = reinterpret_cast(pointer); + + auto pointer = indexVectors->pointerCpu(type); + - auto pdfs = block->getData>(pdfsID); + uint8_t * _data_indexVector = reinterpret_cast(pointer); - WALBERLA_ASSERT_GREATER_EQUAL(0, -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = pdfs->dataAt(0, 0, 0, 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_dynamic_ubb_boundary_Dynamic_UBB::dynamic_ubb_boundary_Dynamic_UBB( - _data_indexVector, _data_pdfs, _stride_pdfs_0, _stride_pdfs_1, - _stride_pdfs_2, _stride_pdfs_3, indexVectorSize); + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + + WALBERLA_ASSERT_GREATER_EQUAL(0, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(0, 0, 0, 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_dynamic_ubb_boundary_Dynamic_UBB::dynamic_ubb_boundary_Dynamic_UBB(_data_indexVector, _data_pdfs, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, indexVectorSize); } -void Dynamic_UBB::run(IBlock *block) { run_impl(block, IndexVectors::ALL); } +void Dynamic_UBB::run(IBlock * block) +{ + run_impl(block, IndexVectors::ALL); +} -void Dynamic_UBB::inner(IBlock *block) { run_impl(block, IndexVectors::INNER); } +void Dynamic_UBB::inner(IBlock * block) +{ + run_impl(block, IndexVectors::INNER); +} -void Dynamic_UBB::outer(IBlock *block) { run_impl(block, IndexVectors::OUTER); } +void Dynamic_UBB::outer(IBlock * block) +{ + run_impl(block, IndexVectors::OUTER); +} } // namespace lbm } // namespace walberla + diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp index e4dcff71f1b..99db3db7b9a 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.cpp @@ -18,1751 +18,1185 @@ #include -#include "FluctuatingMRTLatticeModel.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "FluctuatingMRTLatticeModel.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + #include "philox_rand.h" + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_0, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, - uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, - double kT, double omega_bulk, double omega_even, double omega_odd, - double omega_shear, uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) { - - float random_3_0; - float random_3_1; - float random_3_2; - float random_3_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, - random_3_2, random_3_3); - - float random_2_0; - float random_2_1; - float random_2_2; - float random_2_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, - random_2_2, random_2_3); - - float random_1_0; - float random_1_1; - float random_1_2; - float random_1_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, - random_1_2, random_1_3); - - float random_0_0; - float random_0_1; - float random_0_2; - float random_0_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, - random_0_2, random_0_3); - - const double xi_0 = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_1 = - xi_0 + _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_2 = _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_3 = _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_4 = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_5 = _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_6 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_8 = - -_data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_9 = - xi_8 - - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_10 = - -_data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_11 = - -_data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_12 = - -_data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -_data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - const double xi_15 = - -_data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -_data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_18 = -_data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = - -_data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -_data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_23 = -_data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_24 = xi_17 + xi_22 + xi_23 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_42 = - 0.166666666666667 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_50 = - 0.166666666666667 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_54 = - 0.166666666666667 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_57 = - 0.5 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_61 = - 0.0833333333333333 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_65 = - 0.0833333333333333 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_75 = - 0.0833333333333333 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_93 = -_data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_94 = xi_93 + - 3.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_95 = - omega_even * - (xi_94 - 3.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_96 = - 2.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_97 = - xi_96 + - 5.0 * _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_98 = - omega_even * - (xi_94 + xi_97 - - 2.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] - - 2.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] - - 5.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 5.0 * - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - - _stride_pdfs_0]); - const double xi_101 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_102 = xi_101 + xi_18; - const double xi_103 = - -_data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_106 = - -_data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_107 = xi_106 + xi_11 + xi_15 + xi_21; - const double xi_109 = - 2.0 * - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_110 = - 2.0 * - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_111 = - 2.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 2.0 * _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_112 = - omega_even * - (xi_109 + xi_110 + xi_111 + xi_93 + xi_97 - - 4.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] - - 4.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0] - - 7.0 * - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 5.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_113 = - xi_101 + _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_114 = xi_113 + xi_14 + xi_22 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_116 = xi_114 * xi_115; - const double xi_118 = - xi_103 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_122 = random_2_3 - 0.5; - const double xi_127 = - 2.0 * _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_128 = - 2.0 * _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_129 = - -2.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - 2.0 * _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_130 = -xi_127 + xi_128 + xi_129 + xi_14 + xi_19 + xi_2; - const double xi_132 = xi_130 * xi_131; - const double xi_133 = random_1_2 - 0.5; - const double xi_138 = random_0_1 - 0.5; - const double xi_142 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_156 = - xi_106 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_157 = - xi_12 + xi_156 + xi_20 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_158 = xi_115 * xi_157; - const double xi_159 = random_2_1 - 0.5; - const double xi_161 = xi_1 + xi_127 - xi_128 + xi_129 + xi_13; - const double xi_162 = xi_131 * xi_161; - const double xi_163 = random_2_0 - 0.5; - const double xi_168 = _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_169 = xi_102 + xi_168 + xi_23 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_170 = xi_115 * xi_169; - const double xi_173 = random_2_2 - 0.5; - const double xi_175 = -xi_109 - xi_110 + xi_111 + xi_24 + xi_3; - const double xi_176 = xi_131 * xi_175; - const double xi_177 = random_1_3 - 0.5; - const double xi_184 = xi_112 * 0.0138888888888889; - const double xi_205 = xi_98 * -0.00714285714285714; - const double xi_207 = xi_95 * 0.025; - const double xi_212 = xi_175 * xi_211; - const double xi_214 = xi_169 * xi_213; - const double xi_223 = xi_130 * xi_211; - const double xi_224 = xi_114 * xi_213; - const double xi_232 = xi_98 * 0.0178571428571429; - const double xi_238 = xi_157 * xi_213; - const double xi_239 = xi_161 * xi_211; - const double vel0Term = - xi_1 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double vel1Term = - xi_2 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double vel2Term = - xi_3 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double rho = vel0Term + vel1Term + vel2Term + xi_4 + xi_5 + xi_6 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_7 = 1 / (rho); - const double xi_86 = kT * rho; - const double xi_87 = - sqrt(xi_86 * (-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0)); - const double xi_88 = xi_87 * (random_3_0 - 0.5) * 3.7416573867739413; - const double xi_89 = xi_87 * (random_3_2 - 0.5) * 5.4772255750516612; - const double xi_91 = - xi_90 * sqrt(xi_86 * (-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0)) * - (random_1_1 - 0.5); - const double xi_92 = xi_87 * (random_3_1 - 0.5) * 8.3666002653407556; - const double xi_123 = - sqrt(xi_86 * (-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0)); - const double xi_124 = xi_123 * 1.4142135623730951; - const double xi_125 = xi_124 * 0.5; - const double xi_126 = xi_122 * xi_125; - const double xi_134 = xi_123 * xi_90; - const double xi_135 = xi_134 * 0.166666666666667; - const double xi_136 = xi_133 * xi_135; - const double xi_137 = -xi_132 - xi_136; - const double xi_139 = - sqrt(xi_86 * (-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0)); - const double xi_140 = xi_139 * 0.5; - const double xi_141 = xi_138 * xi_140; - const double xi_146 = - xi_112 * -0.0198412698412698 + xi_88 * -0.119047619047619; - const double xi_148 = xi_139 * (random_0_0 - 0.5) * 1.7320508075688772; - const double xi_152 = xi_132 + xi_136; - const double xi_160 = xi_125 * xi_159; - const double xi_164 = xi_135 * xi_163; - const double xi_165 = xi_162 + xi_164; - const double xi_167 = -xi_162 - xi_164; - const double xi_174 = xi_125 * xi_173; - const double xi_178 = xi_135 * xi_177; - const double xi_179 = -xi_176 - xi_178; - const double xi_181 = xi_176 + xi_178; - const double xi_182 = xi_138 * xi_139 * 0.25; - const double xi_185 = xi_88 * 0.0833333333333333; - const double xi_195 = xi_140 * (random_0_2 - 0.5); - const double xi_204 = xi_140 * (random_1_0 - 0.5); - const double xi_208 = xi_92 * -0.0142857142857143; - const double xi_209 = xi_89 * 0.05; - const double xi_215 = xi_134 * 0.0833333333333333; - const double xi_216 = xi_177 * xi_215; - const double xi_217 = xi_124 * 0.25; - const double xi_218 = xi_173 * xi_217; - const double xi_220 = - xi_112 * -0.00396825396825397 + xi_88 * -0.0238095238095238; - const double xi_225 = xi_133 * xi_215; - const double xi_226 = xi_122 * xi_217; - const double xi_230 = -xi_182; - const double xi_233 = xi_92 * 0.0357142857142857; - const double xi_235 = xi_140 * (random_0_3 - 0.5); - const double xi_240 = xi_159 * xi_217; - const double xi_241 = xi_163 * xi_215; - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_26 = - u_0 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_27 = xi_26 * 0.333333333333333; - const double xi_33 = -xi_27; - const double xi_99 = rho * (u_0 * u_0); - const double xi_153 = rho * u_0; - const double xi_154 = - -vel0Term + xi_142 + xi_153 + xi_4 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_155 = xi_120 * xi_154; - const double xi_191 = xi_154 * xi_186; - const double u_1 = - xi_7 * - (vel1Term + xi_16 + xi_19 + xi_8 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_28 = - u_1 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_29 = xi_28 * 0.333333333333333; - const double xi_34 = -xi_29; - const double xi_56 = u_1 * 0.5; - const double xi_59 = - xi_58 * (u_0 * xi_57 + - xi_56 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_60 = -xi_59; - const double xi_104 = rho * (u_1 * u_1); - const double xi_105 = xi_103 + xi_104 + xi_9; - const double xi_117 = rho * u_1; - const double xi_119 = - -vel1Term + xi_117 + xi_118 + xi_5 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_121 = xi_119 * xi_120; - const double xi_187 = xi_119 * xi_186; - const double xi_197 = - xi_196 * - (u_0 * xi_117 + xi_118 + xi_8 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]); - const double xi_198 = -xi_195 - xi_197; - const double xi_199 = xi_195 + xi_197; - const double u_2 = - xi_7 * - (vel2Term + xi_21 + xi_24 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_30 = - u_2 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_31 = xi_30 * 0.333333333333333; - const double xi_32 = (xi_25 + 2.0) * (xi_27 + xi_29 + xi_31); - const double xi_35 = xi_30 * 0.666666666666667 + xi_33 + xi_34; - const double xi_39 = -xi_31; - const double xi_40 = xi_28 * 0.666666666666667 + xi_33 + xi_39; - const double xi_41 = xi_26 * 0.666666666666667 + xi_34 + xi_39; - const double xi_44 = xi_35 * xi_43; - const double xi_45 = -xi_44; - const double xi_46 = xi_41 * xi_43; - const double xi_47 = -xi_46; - const double xi_49 = xi_40 * xi_48 + xi_45 + xi_47; - const double xi_51 = xi_40 * xi_43; - const double xi_52 = -xi_51; - const double xi_53 = xi_41 * xi_48 + xi_45 + xi_52; - const double xi_55 = xi_35 * xi_48 + xi_47 + xi_52; - const double xi_62 = xi_46 - xi_61; - const double xi_64 = -xi_35 * xi_63; - const double xi_66 = xi_32 * 0.125; - const double xi_67 = xi_51 + xi_66; - const double xi_68 = xi_65 + xi_67; - const double xi_69 = xi_64 + xi_68; - const double xi_70 = xi_46 + xi_61; - const double xi_71 = -xi_65 + xi_67; - const double xi_72 = xi_64 + xi_71; - const double xi_73 = - xi_58 * (u_2 * xi_57 + - xi_56 * _data_force_20_32_10[_stride_force_0 * ctr_0]); - const double xi_74 = -xi_41 * xi_63; - const double xi_76 = xi_44 + xi_75; - const double xi_77 = xi_74 + xi_76; - const double xi_78 = -xi_73; - const double xi_79 = - xi_58 * (u_0 * 0.5 * _data_force_20_32_10[_stride_force_0 * ctr_0] + - u_2 * 0.5 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_80 = -xi_79; - const double xi_81 = -xi_40 * xi_63; - const double xi_82 = xi_66 + xi_76 + xi_81; - const double xi_83 = xi_44 - xi_75; - const double xi_84 = xi_74 + xi_83; - const double xi_85 = xi_66 + xi_81 + xi_83; - const double xi_100 = rho * (u_2 * u_2); - const double xi_108 = - omega_bulk * (xi_100 + xi_102 + xi_105 + xi_107 + xi_17 + xi_22 + - xi_99 + _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]); - const double xi_143 = -xi_100 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_144 = - omega_shear * (xi_0 + xi_105 + xi_142 + xi_143 + xi_16 - - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_145 = xi_144 * 0.125; - const double xi_147 = - omega_shear * - (xi_103 - xi_104 + xi_107 + xi_143 + xi_9 + xi_96 + xi_99 * 2.0 - - 2.0 * - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 2.0 * - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_149 = - xi_147 * -0.0416666666666667 + xi_148 * -0.166666666666667; - const double xi_150 = xi_149 + xi_89 * -0.1 + xi_95 * -0.05; - const double xi_151 = xi_141 + xi_145 + xi_146 + xi_150 + - xi_92 * 0.0285714285714286 + - xi_98 * 0.0142857142857143; - const double xi_166 = - xi_146 + xi_147 * 0.0833333333333333 + xi_148 * 0.333333333333333 + - xi_92 * -0.0714285714285714 + xi_98 * -0.0357142857142857; - const double xi_171 = - rho * u_2 - vel2Term + xi_101 + xi_106 + xi_168 + xi_6 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_172 = xi_120 * xi_171; - const double xi_180 = xi_112 * 0.0158730158730159 - xi_141 - xi_145 + - xi_150 + xi_88 * 0.0952380952380952 + - xi_92 * -0.0428571428571429 + - xi_98 * -0.0214285714285714; - const double xi_183 = xi_144 * 0.0625; - const double xi_188 = - xi_108 * 0.0416666666666667 + xi_91 * 0.0833333333333333; - const double xi_189 = xi_187 + xi_188; - const double xi_190 = - xi_152 + xi_182 + xi_183 + xi_184 + xi_185 + xi_189; - const double xi_192 = - xi_147 * 0.0208333333333333 + xi_148 * 0.0833333333333333; - const double xi_193 = -xi_191 + xi_192; - const double xi_194 = xi_167 + xi_193; - const double xi_200 = xi_191 + xi_192; - const double xi_201 = xi_165 + xi_200; - const double xi_202 = -xi_187 + xi_188; - const double xi_203 = - xi_137 + xi_182 + xi_183 + xi_184 + xi_185 + xi_202; - const double xi_206 = - xi_196 * (u_2 * xi_117 + xi_113 + xi_17 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_210 = - xi_149 + xi_204 + xi_205 + xi_206 + xi_207 + xi_208 + xi_209; - const double xi_219 = xi_171 * xi_186; - const double xi_221 = xi_219 + xi_220; - const double xi_222 = -xi_212 + xi_214 - xi_216 + xi_218 + xi_221; - const double xi_227 = xi_189 - xi_223 + xi_224 - xi_225 + xi_226; - const double xi_228 = xi_202 + xi_223 - xi_224 + xi_225 - xi_226; - const double xi_229 = - xi_149 - xi_204 + xi_205 - xi_206 + xi_207 + xi_208 + xi_209; - const double xi_231 = -xi_183; - const double xi_234 = - xi_181 + xi_188 + xi_221 + xi_230 + xi_231 + xi_232 + xi_233; - const double xi_236 = - xi_196 * - (u_2 * xi_153 + xi_10 + xi_156 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_237 = -xi_235 - xi_236; - const double xi_242 = xi_193 - xi_238 + xi_239 - xi_240 + xi_241; - const double xi_243 = xi_235 + xi_236; - const double xi_244 = xi_200 + xi_238 - xi_239 + xi_240 - xi_241; - const double xi_245 = -xi_219 + xi_220; - const double xi_246 = xi_212 - xi_214 + xi_216 - xi_218 + xi_245; - const double xi_247 = - xi_179 + xi_188 + xi_230 + xi_231 + xi_232 + xi_233 + xi_245; - const double forceTerm_0 = - xi_32 * -1.5 - xi_35 * xi_38 - xi_38 * xi_40 - xi_38 * xi_41; - const double forceTerm_1 = xi_42 + xi_49; - const double forceTerm_2 = -xi_42 + xi_49; - const double forceTerm_3 = -xi_50 + xi_53; - const double forceTerm_4 = xi_50 + xi_53; - const double forceTerm_5 = xi_54 + xi_55; - const double forceTerm_6 = -xi_54 + xi_55; - const double forceTerm_7 = xi_60 + xi_62 + xi_69; - const double forceTerm_8 = xi_59 + xi_69 + xi_70; - const double forceTerm_9 = xi_59 + xi_62 + xi_72; - const double forceTerm_10 = xi_60 + xi_70 + xi_72; - const double forceTerm_11 = xi_68 + xi_73 + xi_77; - const double forceTerm_12 = xi_71 + xi_77 + xi_78; - const double forceTerm_13 = xi_62 + xi_80 + xi_82; - const double forceTerm_14 = xi_70 + xi_79 + xi_82; - const double forceTerm_15 = xi_68 + xi_78 + xi_84; - const double forceTerm_16 = xi_71 + xi_73 + xi_84; - const double forceTerm_17 = xi_62 + xi_79 + xi_85; - const double forceTerm_18 = xi_70 + xi_80 + xi_85; - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_0 + xi_108 * -0.5 + xi_112 * 0.0238095238095238 + - xi_88 * 0.142857142857143 + xi_89 * 0.2 - xi_91 + - xi_92 * 0.0857142857142857 + xi_95 * 0.1 + - xi_98 * 0.0428571428571429 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_1 - xi_116 + xi_121 - xi_126 + xi_137 + xi_151 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_2 + xi_116 - xi_121 + xi_126 + xi_151 + xi_152 + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_3 - xi_155 + xi_158 + xi_160 + xi_165 + xi_166 + - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_4 + xi_155 - xi_158 - xi_160 + xi_166 + xi_167 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_5 - xi_170 + xi_172 - xi_174 + xi_179 + xi_180 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_6 + xi_170 - xi_172 + xi_174 + xi_180 + xi_181 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_7 + xi_190 + xi_194 + xi_198 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_8 + xi_190 + xi_199 + xi_201 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_9 + xi_194 + xi_199 + xi_203 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_10 + xi_198 + xi_201 + xi_203 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_11 + xi_210 + xi_222 + xi_227 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_12 + xi_222 + xi_228 + xi_229 + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_13 + xi_234 + xi_237 + xi_242 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_14 + xi_234 + xi_243 + xi_244 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_15 + xi_227 + xi_229 + xi_246 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_16 + xi_210 + xi_228 + xi_246 + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_17 + xi_242 + xi_243 + xi_247 + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_18 + xi_237 + xi_244 + xi_247 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) + { + + float random_3_0; + float random_3_1; + float random_3_2; + float random_3_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, random_3_2, random_3_3); + + + float random_2_0; + float random_2_1; + float random_2_2; + float random_2_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, random_2_2, random_2_3); + + + float random_1_0; + float random_1_1; + float random_1_2; + float random_1_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, random_1_2, random_1_3); + + + float random_0_0; + float random_0_1; + float random_0_2; + float random_0_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, random_0_2, random_0_3); + + const double xi_0 = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_1 = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_2 = xi_0 + xi_1 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_3 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_4 = xi_3 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + const double xi_5 = xi_4 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_6 = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_7 = xi_6 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_8 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_9 = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_10 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_12 = -_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_13 = -_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_16 = -_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_17 = -_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_21 = xi_20 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_22 = -_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_23 = -_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + const double xi_24 = -_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + const double xi_29 = -_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_30 = xi_24 + xi_28 + xi_29 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_31 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_32 = xi_31 + xi_8 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_33 = xi_16 + xi_21 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_34 = xi_26 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_35 = xi_15 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_9 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_38 = xi_28 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_39 = xi_22 + xi_38 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_57 = 0.166666666666667*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_65 = 0.166666666666667*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_69 = 0.166666666666667*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_72 = 0.5*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_76 = 0.0833333333333333*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_80 = 0.0833333333333333*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_90 = 0.0833333333333333*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_108 = -_data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_109 = xi_108 + 3.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_110 = omega_even*(xi_109 - 3.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_111 = 2.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_112 = xi_111 + 5.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_113 = omega_even*(xi_109 + xi_112 - 2.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] - 2.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] - 5.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 5.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_116 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_117 = xi_116 + xi_22; + const double xi_118 = -_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_121 = -_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_122 = xi_121 + xi_19 + xi_27; + const double xi_124 = 2.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_125 = 2.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_126 = 2.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 2.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_127 = omega_even*(xi_108 + xi_112 + xi_124 + xi_125 + xi_126 - 4.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] - 4.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0] - 7.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 7.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 5.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_128 = xi_116 + xi_23 + xi_38 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_130 = xi_128*xi_129; + const double xi_135 = random_2_3 - 0.5; + const double xi_140 = 2.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_141 = 2.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_142 = -2.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + 2.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_143 = -xi_140 + xi_141 + xi_142 + xi_25 + xi_4; + const double xi_145 = xi_143*xi_144; + const double xi_146 = random_1_2 - 0.5; + const double xi_151 = random_0_1 - 0.5; + const double xi_168 = xi_121 + xi_13 + xi_34 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_169 = xi_129*xi_168; + const double xi_170 = random_2_1 - 0.5; + const double xi_172 = xi_14 + xi_140 - xi_141 + xi_142 + xi_35 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_173 = xi_144*xi_172; + const double xi_174 = random_2_0 - 0.5; + const double xi_179 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_180 = xi_117 + xi_179 + xi_29 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_181 = xi_129*xi_180; + const double xi_184 = random_2_2 - 0.5; + const double xi_186 = -xi_124 - xi_125 + xi_126 + xi_30 + xi_6; + const double xi_187 = xi_144*xi_186; + const double xi_188 = random_1_3 - 0.5; + const double xi_195 = xi_127*0.0138888888888889; + const double xi_216 = xi_113*-0.00714285714285714; + const double xi_218 = xi_110*0.025; + const double xi_223 = xi_186*xi_222; + const double xi_225 = xi_180*xi_224; + const double xi_234 = xi_143*xi_222; + const double xi_235 = xi_128*xi_224; + const double xi_243 = xi_113*0.0178571428571429; + const double xi_249 = xi_168*xi_224; + const double xi_250 = xi_172*xi_222; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_8 + xi_9 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_11 = 1 / (rho); + const double xi_101 = kT*rho; + const double xi_102 = sqrt(xi_101*(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0)); + const double xi_103 = xi_102*(random_3_0 - 0.5)*3.7416573867739413; + const double xi_104 = xi_102*(random_3_2 - 0.5)*5.4772255750516612; + const double xi_106 = xi_105*sqrt(xi_101*(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))*(random_1_1 - 0.5); + const double xi_107 = xi_102*(random_3_1 - 0.5)*8.3666002653407556; + const double xi_136 = sqrt(xi_101*(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0)); + const double xi_137 = xi_136*1.4142135623730951; + const double xi_138 = xi_137*0.5; + const double xi_139 = xi_135*xi_138; + const double xi_147 = xi_105*xi_136; + const double xi_148 = xi_147*0.166666666666667; + const double xi_149 = xi_146*xi_148; + const double xi_150 = -xi_145 - xi_149; + const double xi_152 = sqrt(xi_101*(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0)); + const double xi_153 = xi_152*0.5; + const double xi_154 = xi_151*xi_153; + const double xi_158 = xi_103*-0.119047619047619 + xi_127*-0.0198412698412698; + const double xi_160 = xi_152*(random_0_0 - 0.5)*1.7320508075688772; + const double xi_164 = xi_145 + xi_149; + const double xi_171 = xi_138*xi_170; + const double xi_175 = xi_148*xi_174; + const double xi_176 = xi_173 + xi_175; + const double xi_178 = -xi_173 - xi_175; + const double xi_185 = xi_138*xi_184; + const double xi_189 = xi_148*xi_188; + const double xi_190 = -xi_187 - xi_189; + const double xi_192 = xi_187 + xi_189; + const double xi_193 = xi_151*xi_152*0.25; + const double xi_196 = xi_103*0.0833333333333333; + const double xi_206 = xi_153*(random_0_2 - 0.5); + const double xi_215 = xi_153*(random_1_0 - 0.5); + const double xi_219 = xi_107*-0.0142857142857143; + const double xi_220 = xi_104*0.05; + const double xi_226 = xi_147*0.0833333333333333; + const double xi_227 = xi_188*xi_226; + const double xi_228 = xi_137*0.25; + const double xi_229 = xi_184*xi_228; + const double xi_231 = xi_103*-0.0238095238095238 + xi_127*-0.00396825396825397; + const double xi_236 = xi_146*xi_226; + const double xi_237 = xi_135*xi_228; + const double xi_241 = -xi_193; + const double xi_244 = xi_107*0.0357142857142857; + const double xi_246 = xi_153*(random_0_3 - 0.5); + const double xi_251 = xi_170*xi_228; + const double xi_252 = xi_174*xi_226; + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_41 = u_0*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_42 = xi_41*0.333333333333333; + const double xi_48 = -xi_42; + const double xi_114 = rho*(u_0*u_0); + const double xi_165 = rho*u_0; + const double xi_166 = -vel0Term + xi_165 + xi_32; + const double xi_167 = xi_133*xi_166; + const double xi_202 = xi_166*xi_197; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_43 = u_1*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_44 = xi_43*0.333333333333333; + const double xi_49 = -xi_44; + const double xi_71 = u_1*0.5; + const double xi_74 = xi_73*(u_0*xi_72 + xi_71*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_75 = -xi_74; + const double xi_119 = rho*(u_1*u_1); + const double xi_120 = xi_118 + xi_119 + xi_20; + const double xi_131 = rho*u_1; + const double xi_132 = -vel1Term + xi_118 + xi_131 + xi_37 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_134 = xi_132*xi_133; + const double xi_198 = xi_132*xi_197; + const double u_2 = xi_11*(vel2Term + xi_27 + xi_30 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_45 = u_2*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_46 = xi_45*0.333333333333333; + const double xi_47 = (xi_40 + 2.0)*(xi_42 + xi_44 + xi_46); + const double xi_50 = xi_45*0.666666666666667 + xi_48 + xi_49; + const double xi_54 = -xi_46; + const double xi_55 = xi_43*0.666666666666667 + xi_48 + xi_54; + const double xi_56 = xi_41*0.666666666666667 + xi_49 + xi_54; + const double xi_59 = xi_50*xi_58; + const double xi_60 = -xi_59; + const double xi_61 = xi_56*xi_58; + const double xi_62 = -xi_61; + const double xi_64 = xi_55*xi_63 + xi_60 + xi_62; + const double xi_66 = xi_55*xi_58; + const double xi_67 = -xi_66; + const double xi_68 = xi_56*xi_63 + xi_60 + xi_67; + const double xi_70 = xi_50*xi_63 + xi_62 + xi_67; + const double xi_77 = xi_61 - xi_76; + const double xi_79 = -xi_50*xi_78; + const double xi_81 = xi_47*0.125; + const double xi_82 = xi_66 + xi_81; + const double xi_83 = xi_80 + xi_82; + const double xi_84 = xi_79 + xi_83; + const double xi_85 = xi_61 + xi_76; + const double xi_86 = -xi_80 + xi_82; + const double xi_87 = xi_79 + xi_86; + const double xi_88 = xi_73*(u_2*xi_72 + xi_71*_data_force_20_32_10[_stride_force_0*ctr_0]); + const double xi_89 = -xi_56*xi_78; + const double xi_91 = xi_59 + xi_90; + const double xi_92 = xi_89 + xi_91; + const double xi_93 = -xi_88; + const double xi_94 = xi_73*(u_0*0.5*_data_force_20_32_10[_stride_force_0*ctr_0] + u_2*0.5*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_95 = -xi_94; + const double xi_96 = -xi_55*xi_78; + const double xi_97 = xi_81 + xi_91 + xi_96; + const double xi_98 = xi_59 - xi_90; + const double xi_99 = xi_89 + xi_98; + const double xi_100 = xi_81 + xi_96 + xi_98; + const double xi_115 = rho*(u_2*u_2); + const double xi_123 = omega_bulk*(xi_114 + xi_115 + xi_117 + xi_120 + xi_122 + xi_24 + xi_28 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]); + const double xi_155 = -xi_115 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_156 = omega_shear*(xi_1 + xi_120 + xi_155 + xi_18 + xi_23 + xi_31 - _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]); + const double xi_157 = xi_156*0.125; + const double xi_159 = omega_shear*(xi_111 + xi_114*2.0 + xi_118 - xi_119 + xi_122 + xi_155 + xi_20 - 2.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 2.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_161 = xi_159*-0.0416666666666667 + xi_160*-0.166666666666667; + const double xi_162 = xi_104*-0.1 + xi_110*-0.05 + xi_161; + const double xi_163 = xi_107*0.0285714285714286 + xi_113*0.0142857142857143 + xi_154 + xi_157 + xi_158 + xi_162; + const double xi_177 = xi_107*-0.0714285714285714 + xi_113*-0.0357142857142857 + xi_158 + xi_159*0.0833333333333333 + xi_160*0.333333333333333; + const double xi_182 = rho*u_2 - vel2Term + xi_10 + xi_116 + xi_121 + xi_179 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_183 = xi_133*xi_182; + const double xi_191 = xi_103*0.0952380952380952 + xi_107*-0.0428571428571429 + xi_113*-0.0214285714285714 + xi_127*0.0158730158730159 - xi_154 - xi_157 + xi_162; + const double xi_194 = xi_156*0.0625; + const double xi_199 = xi_106*0.0833333333333333 + xi_123*0.0416666666666667; + const double xi_200 = xi_198 + xi_199; + const double xi_201 = xi_164 + xi_193 + xi_194 + xi_195 + xi_196 + xi_200; + const double xi_203 = xi_159*0.0208333333333333 + xi_160*0.0833333333333333; + const double xi_204 = -xi_202 + xi_203; + const double xi_205 = xi_178 + xi_204; + const double xi_211 = xi_202 + xi_203; + const double xi_212 = xi_176 + xi_211; + const double xi_213 = -xi_198 + xi_199; + const double xi_214 = xi_150 + xi_193 + xi_194 + xi_195 + xi_196 + xi_213; + const double xi_230 = xi_182*xi_197; + const double xi_232 = xi_230 + xi_231; + const double xi_233 = -xi_223 + xi_225 - xi_227 + xi_229 + xi_232; + const double xi_238 = xi_200 - xi_234 + xi_235 - xi_236 + xi_237; + const double xi_239 = xi_213 + xi_234 - xi_235 + xi_236 - xi_237; + const double xi_242 = -xi_194; + const double xi_245 = xi_192 + xi_199 + xi_232 + xi_241 + xi_242 + xi_243 + xi_244; + const double xi_253 = xi_204 - xi_249 + xi_250 - xi_251 + xi_252; + const double xi_255 = xi_211 + xi_249 - xi_250 + xi_251 - xi_252; + const double xi_256 = -xi_230 + xi_231; + const double xi_257 = xi_223 - xi_225 + xi_227 - xi_229 + xi_256; + const double xi_258 = xi_190 + xi_199 + xi_241 + xi_242 + xi_243 + xi_244 + xi_256; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_208 = xi_207*(-p_1 + u_0*xi_131); + const double xi_209 = -xi_206 - xi_208; + const double xi_210 = xi_206 + xi_208; + const double p_2 = xi_36; + const double xi_247 = xi_207*(-p_2 + u_2*xi_165); + const double xi_248 = -xi_246 - xi_247; + const double xi_254 = xi_246 + xi_247; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_217 = xi_207*(-p_5 + u_2*xi_131); + const double xi_221 = xi_161 + xi_215 + xi_216 + xi_217 + xi_218 + xi_219 + xi_220; + const double xi_240 = xi_161 - xi_215 + xi_216 - xi_217 + xi_218 + xi_219 + xi_220; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_3 + xi_7 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double forceTerm_0 = xi_47*-1.5 - xi_50*xi_53 - xi_53*xi_55 - xi_53*xi_56; + const double forceTerm_1 = xi_57 + xi_64; + const double forceTerm_2 = -xi_57 + xi_64; + const double forceTerm_3 = -xi_65 + xi_68; + const double forceTerm_4 = xi_65 + xi_68; + const double forceTerm_5 = xi_69 + xi_70; + const double forceTerm_6 = -xi_69 + xi_70; + const double forceTerm_7 = xi_75 + xi_77 + xi_84; + const double forceTerm_8 = xi_74 + xi_84 + xi_85; + const double forceTerm_9 = xi_74 + xi_77 + xi_87; + const double forceTerm_10 = xi_75 + xi_85 + xi_87; + const double forceTerm_11 = xi_83 + xi_88 + xi_92; + const double forceTerm_12 = xi_86 + xi_92 + xi_93; + const double forceTerm_13 = xi_77 + xi_95 + xi_97; + const double forceTerm_14 = xi_85 + xi_94 + xi_97; + const double forceTerm_15 = xi_83 + xi_93 + xi_99; + const double forceTerm_16 = xi_86 + xi_88 + xi_99; + const double forceTerm_17 = xi_100 + xi_77 + xi_94; + const double forceTerm_18 = xi_100 + xi_85 + xi_95; + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_0 + xi_103*0.142857142857143 + xi_104*0.2 - xi_106 + xi_107*0.0857142857142857 + xi_110*0.1 + xi_113*0.0428571428571429 + xi_123*-0.5 + xi_127*0.0238095238095238 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_1 - xi_130 + xi_134 - xi_139 + xi_150 + xi_163 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_2 + xi_130 - xi_134 + xi_139 + xi_163 + xi_164 + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_3 - xi_167 + xi_169 + xi_171 + xi_176 + xi_177 + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_4 + xi_167 - xi_169 - xi_171 + xi_177 + xi_178 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_5 - xi_181 + xi_183 - xi_185 + xi_190 + xi_191 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_6 + xi_181 - xi_183 + xi_185 + xi_191 + xi_192 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_7 + xi_201 + xi_205 + xi_209 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_8 + xi_201 + xi_210 + xi_212 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_9 + xi_205 + xi_210 + xi_214 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_10 + xi_209 + xi_212 + xi_214 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_11 + xi_221 + xi_233 + xi_238 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_12 + xi_233 + xi_239 + xi_240 + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_13 + xi_245 + xi_248 + xi_253 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_14 + xi_245 + xi_254 + xi_255 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_15 + xi_238 + xi_240 + xi_257 + _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_16 + xi_221 + xi_239 + xi_257 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_17 + xi_253 + xi_254 + xi_258 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_18 + xi_248 + xi_255 + xi_258 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_0, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, uint32_t block_offset_0, - uint32_t block_offset_1, uint32_t block_offset_2, double kT, - double omega_bulk, double omega_even, double omega_odd, - double omega_shear, uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; - for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) { - const double xi_248 = _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0]; - const double xi_249 = _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0]; - const double xi_250 = _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0]; - const double xi_251 = _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0]; - const double xi_252 = _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_253 = _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0]; - const double xi_254 = _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0]; - const double xi_255 = _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_256 = _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0]; - const double xi_257 = _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0]; - const double xi_258 = _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0]; - const double xi_259 = _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_260 = _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_261 = _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_262 = _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0]; - const double xi_263 = _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0]; - const double xi_264 = _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0]; - const double xi_265 = _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0]; - const double xi_266 = _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0]; - const double xi_267 = _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0]; - const double xi_268 = _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_269 = _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0]; - - float random_3_0; - float random_3_1; - float random_3_2; - float random_3_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, - random_3_2, random_3_3); - - float random_2_0; - float random_2_1; - float random_2_2; - float random_2_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, - random_2_2, random_2_3); - - float random_1_0; - float random_1_1; - float random_1_2; - float random_1_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, - random_1_2, random_1_3); - - float random_0_0; - float random_0_1; - float random_0_2; - float random_0_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, - random_0_2, random_0_3); - - const double xi_0 = xi_256 + xi_262; - const double xi_1 = xi_0 + xi_253; - const double xi_2 = xi_250 + xi_264 + xi_265; - const double xi_3 = xi_248 + xi_259; - const double xi_4 = xi_257 + xi_258; - const double xi_5 = xi_249 + xi_269; - const double xi_6 = xi_260 + xi_266; - const double xi_8 = -xi_258; - const double xi_9 = -xi_254 + xi_8; - const double xi_10 = -xi_266; - const double xi_11 = -xi_251; - const double xi_12 = -xi_257; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -xi_269; - const double xi_15 = -xi_267; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -xi_249; - const double xi_18 = -xi_248; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = -xi_262; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -xi_265; - const double xi_23 = -xi_260; - const double xi_24 = xi_17 + xi_22 + xi_23 + xi_264; - const double xi_42 = xi_255 * 0.166666666666667; - const double xi_50 = xi_261 * 0.166666666666667; - const double xi_54 = xi_268 * 0.166666666666667; - const double xi_57 = xi_255 * 0.5; - const double xi_61 = xi_261 * 0.0833333333333333; - const double xi_65 = xi_255 * 0.0833333333333333; - const double xi_75 = xi_268 * 0.0833333333333333; - const double xi_93 = -xi_252; - const double xi_94 = xi_259 * 3.0 + xi_260 * 3.0 + xi_93; - const double xi_95 = - omega_even * (xi_248 * -3.0 + xi_249 * -3.0 + xi_250 * 3.0 + - xi_264 * -3.0 + xi_265 * -3.0 + xi_269 * 3.0 + xi_94); - const double xi_96 = - xi_248 * 2.0 + xi_249 * 2.0 + xi_264 * 2.0 + xi_265 * 2.0; - const double xi_97 = xi_253 * 5.0 + xi_257 * 5.0 + xi_96; - const double xi_98 = - omega_even * - (xi_250 * -2.0 + xi_251 * -5.0 + xi_256 * -5.0 + xi_262 * -5.0 + - xi_266 * -5.0 + xi_269 * -2.0 + xi_94 + xi_97); - const double xi_101 = -xi_264; - const double xi_102 = xi_101 + xi_18; - const double xi_103 = -xi_263; - const double xi_106 = -xi_256; - const double xi_107 = xi_106 + xi_11 + xi_15 + xi_21; - const double xi_109 = xi_251 * 2.0; - const double xi_110 = xi_256 * 2.0; - const double xi_111 = xi_262 * 2.0 + xi_266 * 2.0; - const double xi_112 = - omega_even * - (xi_109 + xi_110 + xi_111 + xi_250 * 5.0 + xi_254 * -7.0 + - xi_258 * -7.0 + xi_259 * -4.0 + xi_260 * -4.0 + xi_263 * -7.0 + - xi_267 * -7.0 + xi_269 * 5.0 + xi_93 + xi_97); - const double xi_113 = xi_101 + xi_248; - const double xi_114 = xi_113 + xi_14 + xi_22 + xi_249 + xi_250; - const double xi_116 = xi_114 * xi_115; - const double xi_118 = xi_103 + xi_267; - const double xi_122 = random_2_3 - 0.5; - const double xi_127 = xi_254 * 2.0; - const double xi_128 = xi_267 * 2.0; - const double xi_129 = xi_258 * 2.0 + xi_263 * -2.0; - const double xi_130 = -xi_127 + xi_128 + xi_129 + xi_14 + xi_19 + xi_2; - const double xi_132 = xi_130 * xi_131; - const double xi_133 = random_1_2 - 0.5; - const double xi_138 = random_0_1 - 0.5; - const double xi_142 = xi_251 + xi_266; - const double xi_156 = xi_106 + xi_251; - const double xi_157 = xi_12 + xi_156 + xi_20 + xi_253 + xi_266; - const double xi_158 = xi_115 * xi_157; - const double xi_159 = random_2_1 - 0.5; - const double xi_161 = xi_1 + xi_127 - xi_128 + xi_129 + xi_13; - const double xi_162 = xi_131 * xi_161; - const double xi_163 = random_2_0 - 0.5; - const double xi_168 = xi_249 + xi_265; - const double xi_169 = xi_102 + xi_168 + xi_23 + xi_259; - const double xi_170 = xi_115 * xi_169; - const double xi_173 = random_2_2 - 0.5; - const double xi_175 = -xi_109 - xi_110 + xi_111 + xi_24 + xi_3; - const double xi_176 = xi_131 * xi_175; - const double xi_177 = random_1_3 - 0.5; - const double xi_184 = xi_112 * 0.0138888888888889; - const double xi_205 = xi_98 * -0.00714285714285714; - const double xi_207 = xi_95 * 0.025; - const double xi_212 = xi_175 * xi_211; - const double xi_214 = xi_169 * xi_213; - const double xi_223 = xi_130 * xi_211; - const double xi_224 = xi_114 * xi_213; - const double xi_232 = xi_98 * 0.0178571428571429; - const double xi_238 = xi_157 * xi_213; - const double xi_239 = xi_161 * xi_211; - const double vel0Term = xi_1 + xi_263 + xi_267; - const double vel1Term = xi_2 + xi_254; - const double vel2Term = xi_251 + xi_3; - const double rho = - vel0Term + vel1Term + vel2Term + xi_252 + xi_4 + xi_5 + xi_6; - const double xi_7 = 1 / (rho); - const double xi_86 = kT * rho; - const double xi_87 = - sqrt(xi_86 * (-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0)); - const double xi_88 = xi_87 * (random_3_0 - 0.5) * 3.7416573867739413; - const double xi_89 = xi_87 * (random_3_2 - 0.5) * 5.4772255750516612; - const double xi_91 = - xi_90 * sqrt(xi_86 * (-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0)) * - (random_1_1 - 0.5); - const double xi_92 = xi_87 * (random_3_1 - 0.5) * 8.3666002653407556; - const double xi_123 = - sqrt(xi_86 * (-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0)); - const double xi_124 = xi_123 * 1.4142135623730951; - const double xi_125 = xi_124 * 0.5; - const double xi_126 = xi_122 * xi_125; - const double xi_134 = xi_123 * xi_90; - const double xi_135 = xi_134 * 0.166666666666667; - const double xi_136 = xi_133 * xi_135; - const double xi_137 = -xi_132 - xi_136; - const double xi_139 = - sqrt(xi_86 * (-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0)); - const double xi_140 = xi_139 * 0.5; - const double xi_141 = xi_138 * xi_140; - const double xi_146 = - xi_112 * -0.0198412698412698 + xi_88 * -0.119047619047619; - const double xi_148 = xi_139 * (random_0_0 - 0.5) * 1.7320508075688772; - const double xi_152 = xi_132 + xi_136; - const double xi_160 = xi_125 * xi_159; - const double xi_164 = xi_135 * xi_163; - const double xi_165 = xi_162 + xi_164; - const double xi_167 = -xi_162 - xi_164; - const double xi_174 = xi_125 * xi_173; - const double xi_178 = xi_135 * xi_177; - const double xi_179 = -xi_176 - xi_178; - const double xi_181 = xi_176 + xi_178; - const double xi_182 = xi_138 * xi_139 * 0.25; - const double xi_185 = xi_88 * 0.0833333333333333; - const double xi_195 = xi_140 * (random_0_2 - 0.5); - const double xi_204 = xi_140 * (random_1_0 - 0.5); - const double xi_208 = xi_92 * -0.0142857142857143; - const double xi_209 = xi_89 * 0.05; - const double xi_215 = xi_134 * 0.0833333333333333; - const double xi_216 = xi_177 * xi_215; - const double xi_217 = xi_124 * 0.25; - const double xi_218 = xi_173 * xi_217; - const double xi_220 = - xi_112 * -0.00396825396825397 + xi_88 * -0.0238095238095238; - const double xi_225 = xi_133 * xi_215; - const double xi_226 = xi_122 * xi_217; - const double xi_230 = -xi_182; - const double xi_233 = xi_92 * 0.0357142857142857; - const double xi_235 = xi_140 * (random_0_3 - 0.5); - const double xi_240 = xi_159 * xi_217; - const double xi_241 = xi_163 * xi_215; - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_26 = u_0 * xi_261; - const double xi_27 = xi_26 * 0.333333333333333; - const double xi_33 = -xi_27; - const double xi_99 = rho * (u_0 * u_0); - const double xi_153 = rho * u_0; - const double xi_154 = -vel0Term + xi_142 + xi_153 + xi_254 + xi_4; - const double xi_155 = xi_120 * xi_154; - const double xi_191 = xi_154 * xi_186; - const double u_1 = xi_7 * (vel1Term + xi_16 + xi_19 + xi_263 + xi_8); - const double xi_28 = u_1 * xi_255; - const double xi_29 = xi_28 * 0.333333333333333; - const double xi_34 = -xi_29; - const double xi_56 = u_1 * 0.5; - const double xi_59 = xi_58 * (u_0 * xi_57 + xi_261 * xi_56); - const double xi_60 = -xi_59; - const double xi_104 = rho * (u_1 * u_1); - const double xi_105 = xi_103 + xi_104 + xi_9; - const double xi_117 = rho * u_1; - const double xi_119 = - -vel1Term + xi_117 + xi_118 + xi_248 + xi_258 + xi_5; - const double xi_121 = xi_119 * xi_120; - const double xi_187 = xi_119 * xi_186; - const double xi_197 = xi_196 * (u_0 * xi_117 + xi_118 + xi_254 + xi_8); - const double xi_198 = -xi_195 - xi_197; - const double xi_199 = xi_195 + xi_197; - const double u_2 = xi_7 * (vel2Term + xi_21 + xi_24 + xi_256); - const double xi_30 = u_2 * xi_268; - const double xi_31 = xi_30 * 0.333333333333333; - const double xi_32 = (xi_25 + 2.0) * (xi_27 + xi_29 + xi_31); - const double xi_35 = xi_30 * 0.666666666666667 + xi_33 + xi_34; - const double xi_39 = -xi_31; - const double xi_40 = xi_28 * 0.666666666666667 + xi_33 + xi_39; - const double xi_41 = xi_26 * 0.666666666666667 + xi_34 + xi_39; - const double xi_44 = xi_35 * xi_43; - const double xi_45 = -xi_44; - const double xi_46 = xi_41 * xi_43; - const double xi_47 = -xi_46; - const double xi_49 = xi_40 * xi_48 + xi_45 + xi_47; - const double xi_51 = xi_40 * xi_43; - const double xi_52 = -xi_51; - const double xi_53 = xi_41 * xi_48 + xi_45 + xi_52; - const double xi_55 = xi_35 * xi_48 + xi_47 + xi_52; - const double xi_62 = xi_46 - xi_61; - const double xi_64 = -xi_35 * xi_63; - const double xi_66 = xi_32 * 0.125; - const double xi_67 = xi_51 + xi_66; - const double xi_68 = xi_65 + xi_67; - const double xi_69 = xi_64 + xi_68; - const double xi_70 = xi_46 + xi_61; - const double xi_71 = -xi_65 + xi_67; - const double xi_72 = xi_64 + xi_71; - const double xi_73 = xi_58 * (u_2 * xi_57 + xi_268 * xi_56); - const double xi_74 = -xi_41 * xi_63; - const double xi_76 = xi_44 + xi_75; - const double xi_77 = xi_74 + xi_76; - const double xi_78 = -xi_73; - const double xi_79 = xi_58 * (u_0 * xi_268 * 0.5 + u_2 * xi_261 * 0.5); - const double xi_80 = -xi_79; - const double xi_81 = -xi_40 * xi_63; - const double xi_82 = xi_66 + xi_76 + xi_81; - const double xi_83 = xi_44 - xi_75; - const double xi_84 = xi_74 + xi_83; - const double xi_85 = xi_66 + xi_81 + xi_83; - const double xi_100 = rho * (u_2 * u_2); - const double xi_108 = omega_bulk * (xi_100 + xi_102 + xi_105 + xi_107 + - xi_17 + xi_22 + xi_252 + xi_99); - const double xi_143 = -xi_100 + xi_259 + xi_260; - const double xi_144 = - omega_shear * (xi_0 + xi_105 + xi_142 + xi_143 + xi_16 - xi_250); - const double xi_145 = xi_144 * 0.125; - const double xi_147 = - omega_shear * - (xi_103 - xi_104 + xi_107 + xi_143 + xi_250 + xi_253 * -2.0 + - xi_257 * -2.0 + xi_269 + xi_9 + xi_96 + xi_99 * 2.0); - const double xi_149 = - xi_147 * -0.0416666666666667 + xi_148 * -0.166666666666667; - const double xi_150 = xi_149 + xi_89 * -0.1 + xi_95 * -0.05; - const double xi_151 = xi_141 + xi_145 + xi_146 + xi_150 + - xi_92 * 0.0285714285714286 + - xi_98 * 0.0142857142857143; - const double xi_166 = - xi_146 + xi_147 * 0.0833333333333333 + xi_148 * 0.333333333333333 + - xi_92 * -0.0714285714285714 + xi_98 * -0.0357142857142857; - const double xi_171 = - rho * u_2 - vel2Term + xi_101 + xi_106 + xi_168 + xi_262 + xi_6; - const double xi_172 = xi_120 * xi_171; - const double xi_180 = xi_112 * 0.0158730158730159 - xi_141 - xi_145 + - xi_150 + xi_88 * 0.0952380952380952 + - xi_92 * -0.0428571428571429 + - xi_98 * -0.0214285714285714; - const double xi_183 = xi_144 * 0.0625; - const double xi_188 = - xi_108 * 0.0416666666666667 + xi_91 * 0.0833333333333333; - const double xi_189 = xi_187 + xi_188; - const double xi_190 = - xi_152 + xi_182 + xi_183 + xi_184 + xi_185 + xi_189; - const double xi_192 = - xi_147 * 0.0208333333333333 + xi_148 * 0.0833333333333333; - const double xi_193 = -xi_191 + xi_192; - const double xi_194 = xi_167 + xi_193; - const double xi_200 = xi_191 + xi_192; - const double xi_201 = xi_165 + xi_200; - const double xi_202 = -xi_187 + xi_188; - const double xi_203 = - xi_137 + xi_182 + xi_183 + xi_184 + xi_185 + xi_202; - const double xi_206 = xi_196 * (u_2 * xi_117 + xi_113 + xi_17 + xi_265); - const double xi_210 = - xi_149 + xi_204 + xi_205 + xi_206 + xi_207 + xi_208 + xi_209; - const double xi_219 = xi_171 * xi_186; - const double xi_221 = xi_219 + xi_220; - const double xi_222 = -xi_212 + xi_214 - xi_216 + xi_218 + xi_221; - const double xi_227 = xi_189 - xi_223 + xi_224 - xi_225 + xi_226; - const double xi_228 = xi_202 + xi_223 - xi_224 + xi_225 - xi_226; - const double xi_229 = - xi_149 - xi_204 + xi_205 - xi_206 + xi_207 + xi_208 + xi_209; - const double xi_231 = -xi_183; - const double xi_234 = - xi_181 + xi_188 + xi_221 + xi_230 + xi_231 + xi_232 + xi_233; - const double xi_236 = xi_196 * (u_2 * xi_153 + xi_10 + xi_156 + xi_262); - const double xi_237 = -xi_235 - xi_236; - const double xi_242 = xi_193 - xi_238 + xi_239 - xi_240 + xi_241; - const double xi_243 = xi_235 + xi_236; - const double xi_244 = xi_200 + xi_238 - xi_239 + xi_240 - xi_241; - const double xi_245 = -xi_219 + xi_220; - const double xi_246 = xi_212 - xi_214 + xi_216 - xi_218 + xi_245; - const double xi_247 = - xi_179 + xi_188 + xi_230 + xi_231 + xi_232 + xi_233 + xi_245; - const double forceTerm_0 = - xi_32 * -1.5 - xi_35 * xi_38 - xi_38 * xi_40 - xi_38 * xi_41; - const double forceTerm_1 = xi_42 + xi_49; - const double forceTerm_2 = -xi_42 + xi_49; - const double forceTerm_3 = -xi_50 + xi_53; - const double forceTerm_4 = xi_50 + xi_53; - const double forceTerm_5 = xi_54 + xi_55; - const double forceTerm_6 = -xi_54 + xi_55; - const double forceTerm_7 = xi_60 + xi_62 + xi_69; - const double forceTerm_8 = xi_59 + xi_69 + xi_70; - const double forceTerm_9 = xi_59 + xi_62 + xi_72; - const double forceTerm_10 = xi_60 + xi_70 + xi_72; - const double forceTerm_11 = xi_68 + xi_73 + xi_77; - const double forceTerm_12 = xi_71 + xi_77 + xi_78; - const double forceTerm_13 = xi_62 + xi_80 + xi_82; - const double forceTerm_14 = xi_70 + xi_79 + xi_82; - const double forceTerm_15 = xi_68 + xi_78 + xi_84; - const double forceTerm_16 = xi_71 + xi_73 + xi_84; - const double forceTerm_17 = xi_62 + xi_79 + xi_85; - const double forceTerm_18 = xi_70 + xi_80 + xi_85; - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0] = - forceTerm_0 + xi_108 * -0.5 + xi_112 * 0.0238095238095238 + xi_252 + - xi_88 * 0.142857142857143 + xi_89 * 0.2 - xi_91 + - xi_92 * 0.0857142857142857 + xi_95 * 0.1 + - xi_98 * 0.0428571428571429; - _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0] = - forceTerm_1 - xi_116 + xi_121 - xi_126 + xi_137 + xi_151 + xi_250; - _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0] = - forceTerm_2 + xi_116 - xi_121 + xi_126 + xi_151 + xi_152 + xi_269; - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0] = - forceTerm_3 - xi_155 + xi_158 + xi_160 + xi_165 + xi_166 + xi_257; - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0] = - forceTerm_4 + xi_155 - xi_158 - xi_160 + xi_166 + xi_167 + xi_253; - _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0] = - forceTerm_5 - xi_170 + xi_172 - xi_174 + xi_179 + xi_180 + xi_259; - _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0] = - forceTerm_6 + xi_170 - xi_172 + xi_174 + xi_180 + xi_181 + xi_260; - _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0] = - forceTerm_7 + xi_190 + xi_194 + xi_198 + xi_254; - _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0] = - forceTerm_8 + xi_190 + xi_199 + xi_201 + xi_263; - _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0] = - forceTerm_9 + xi_194 + xi_199 + xi_203 + xi_258; - _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0] = - forceTerm_10 + xi_198 + xi_201 + xi_203 + xi_267; - _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0] = - forceTerm_11 + xi_210 + xi_222 + xi_227 + xi_264; - _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0] = - forceTerm_12 + xi_222 + xi_228 + xi_229 + xi_248; - _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0] = - forceTerm_13 + xi_234 + xi_237 + xi_242 + xi_251; - _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0] = - forceTerm_14 + xi_234 + xi_243 + xi_244 + xi_256; - _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0] = - forceTerm_15 + xi_227 + xi_229 + xi_246 + xi_265; - _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0] = - forceTerm_16 + xi_210 + xi_228 + xi_246 + xi_249; - _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0] = - forceTerm_17 + xi_242 + xi_243 + xi_247 + xi_266; - _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0] = - forceTerm_18 + xi_237 + xi_244 + xi_247 + xi_262; +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) + { + const double xi_259 = _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0]; + const double xi_260 = _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0]; + const double xi_261 = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0]; + const double xi_262 = _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0]; + const double xi_263 = _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0]; + const double xi_264 = _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0]; + const double xi_265 = _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0]; + const double xi_266 = _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0]; + const double xi_267 = _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0]; + const double xi_268 = _data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_269 = _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0]; + const double xi_270 = _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0]; + const double xi_271 = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_272 = _data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_273 = _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0]; + const double xi_274 = _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0]; + const double xi_275 = _data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_276 = _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0]; + const double xi_277 = _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0]; + const double xi_278 = _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0]; + const double xi_279 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0]; + const double xi_280 = _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0]; + + float random_3_0; + float random_3_1; + float random_3_2; + float random_3_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, random_3_2, random_3_3); + + + float random_2_0; + float random_2_1; + float random_2_2; + float random_2_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, random_2_2, random_2_3); + + + float random_1_0; + float random_1_1; + float random_1_2; + float random_1_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, random_1_2, random_1_3); + + + float random_0_0; + float random_0_1; + float random_0_2; + float random_0_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, random_0_2, random_0_3); + + const double xi_0 = xi_263 + xi_273; + const double xi_1 = xi_260 + xi_266; + const double xi_2 = xi_0 + xi_1 + xi_261; + const double xi_3 = xi_265 + xi_270; + const double xi_4 = xi_276 + xi_3; + const double xi_5 = xi_280 + xi_4; + const double xi_6 = xi_274 + xi_277; + const double xi_7 = xi_267 + xi_6; + const double xi_8 = xi_269 + xi_279; + const double xi_9 = xi_259 + xi_262; + const double xi_10 = xi_264 + xi_278; + const double xi_12 = -xi_278; + const double xi_13 = -xi_279; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -xi_267; + const double xi_16 = -xi_280; + const double xi_17 = -xi_269; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -xi_273; + const double xi_21 = xi_20 + xi_263; + const double xi_22 = -xi_277; + const double xi_23 = -xi_259; + const double xi_24 = -xi_262; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -xi_266; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -xi_270; + const double xi_29 = -xi_264; + const double xi_30 = xi_24 + xi_265 + xi_28 + xi_29; + const double xi_31 = xi_267 + xi_278; + const double xi_32 = xi_280 + xi_31 + xi_8; + const double xi_33 = xi_16 + xi_21 + xi_269; + const double xi_34 = xi_26 + xi_278; + const double xi_35 = xi_15 + xi_260; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_269 + xi_277 + xi_9; + const double xi_38 = xi_262 + xi_28; + const double xi_39 = xi_22 + xi_265 + xi_38; + const double xi_57 = xi_272*0.166666666666667; + const double xi_65 = xi_268*0.166666666666667; + const double xi_69 = xi_275*0.166666666666667; + const double xi_72 = xi_272*0.5; + const double xi_76 = xi_268*0.0833333333333333; + const double xi_80 = xi_272*0.0833333333333333; + const double xi_90 = xi_275*0.0833333333333333; + const double xi_108 = -xi_271; + const double xi_109 = xi_108 + xi_264*3.0 + xi_274*3.0; + const double xi_110 = omega_even*(xi_109 + xi_259*3.0 + xi_262*-3.0 + xi_265*-3.0 + xi_270*-3.0 + xi_276*3.0 + xi_277*-3.0); + const double xi_111 = xi_262*2.0 + xi_265*2.0 + xi_270*2.0 + xi_277*2.0; + const double xi_112 = xi_111 + xi_261*5.0 + xi_279*5.0; + const double xi_113 = omega_even*(xi_109 + xi_112 + xi_259*-2.0 + xi_260*-5.0 + xi_266*-5.0 + xi_267*-5.0 + xi_276*-2.0 + xi_278*-5.0); + const double xi_116 = -xi_265; + const double xi_117 = xi_116 + xi_22; + const double xi_118 = -xi_263; + const double xi_121 = -xi_260; + const double xi_122 = xi_121 + xi_19 + xi_27; + const double xi_124 = xi_267*2.0; + const double xi_125 = xi_260*2.0; + const double xi_126 = xi_266*2.0 + xi_278*2.0; + const double xi_127 = omega_even*(xi_108 + xi_112 + xi_124 + xi_125 + xi_126 + xi_259*5.0 + xi_263*-7.0 + xi_264*-4.0 + xi_269*-7.0 + xi_273*-7.0 + xi_274*-4.0 + xi_276*5.0 + xi_280*-7.0); + const double xi_128 = xi_116 + xi_23 + xi_276 + xi_277 + xi_38; + const double xi_130 = xi_128*xi_129; + const double xi_135 = random_2_3 - 0.5; + const double xi_140 = xi_280*2.0; + const double xi_141 = xi_273*2.0; + const double xi_142 = xi_263*-2.0 + xi_269*2.0; + const double xi_143 = -xi_140 + xi_141 + xi_142 + xi_25 + xi_4; + const double xi_145 = xi_143*xi_144; + const double xi_146 = random_1_2 - 0.5; + const double xi_151 = random_0_1 - 0.5; + const double xi_168 = xi_121 + xi_13 + xi_261 + xi_267 + xi_34; + const double xi_169 = xi_129*xi_168; + const double xi_170 = random_2_1 - 0.5; + const double xi_172 = xi_14 + xi_140 - xi_141 + xi_142 + xi_261 + xi_266 + xi_35; + const double xi_173 = xi_144*xi_172; + const double xi_174 = random_2_0 - 0.5; + const double xi_179 = xi_262 + xi_270; + const double xi_180 = xi_117 + xi_179 + xi_274 + xi_29; + const double xi_181 = xi_129*xi_180; + const double xi_184 = random_2_2 - 0.5; + const double xi_186 = -xi_124 - xi_125 + xi_126 + xi_30 + xi_6; + const double xi_187 = xi_144*xi_186; + const double xi_188 = random_1_3 - 0.5; + const double xi_195 = xi_127*0.0138888888888889; + const double xi_216 = xi_113*-0.00714285714285714; + const double xi_218 = xi_110*0.025; + const double xi_223 = xi_186*xi_222; + const double xi_225 = xi_180*xi_224; + const double xi_234 = xi_143*xi_222; + const double xi_235 = xi_128*xi_224; + const double xi_243 = xi_113*0.0178571428571429; + const double xi_249 = xi_168*xi_224; + const double xi_250 = xi_172*xi_222; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_271 + xi_8 + xi_9; + const double xi_11 = 1 / (rho); + const double xi_101 = kT*rho; + const double xi_102 = sqrt(xi_101*(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0)); + const double xi_103 = xi_102*(random_3_0 - 0.5)*3.7416573867739413; + const double xi_104 = xi_102*(random_3_2 - 0.5)*5.4772255750516612; + const double xi_106 = xi_105*sqrt(xi_101*(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))*(random_1_1 - 0.5); + const double xi_107 = xi_102*(random_3_1 - 0.5)*8.3666002653407556; + const double xi_136 = sqrt(xi_101*(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0)); + const double xi_137 = xi_136*1.4142135623730951; + const double xi_138 = xi_137*0.5; + const double xi_139 = xi_135*xi_138; + const double xi_147 = xi_105*xi_136; + const double xi_148 = xi_147*0.166666666666667; + const double xi_149 = xi_146*xi_148; + const double xi_150 = -xi_145 - xi_149; + const double xi_152 = sqrt(xi_101*(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0)); + const double xi_153 = xi_152*0.5; + const double xi_154 = xi_151*xi_153; + const double xi_158 = xi_103*-0.119047619047619 + xi_127*-0.0198412698412698; + const double xi_160 = xi_152*(random_0_0 - 0.5)*1.7320508075688772; + const double xi_164 = xi_145 + xi_149; + const double xi_171 = xi_138*xi_170; + const double xi_175 = xi_148*xi_174; + const double xi_176 = xi_173 + xi_175; + const double xi_178 = -xi_173 - xi_175; + const double xi_185 = xi_138*xi_184; + const double xi_189 = xi_148*xi_188; + const double xi_190 = -xi_187 - xi_189; + const double xi_192 = xi_187 + xi_189; + const double xi_193 = xi_151*xi_152*0.25; + const double xi_196 = xi_103*0.0833333333333333; + const double xi_206 = xi_153*(random_0_2 - 0.5); + const double xi_215 = xi_153*(random_1_0 - 0.5); + const double xi_219 = xi_107*-0.0142857142857143; + const double xi_220 = xi_104*0.05; + const double xi_226 = xi_147*0.0833333333333333; + const double xi_227 = xi_188*xi_226; + const double xi_228 = xi_137*0.25; + const double xi_229 = xi_184*xi_228; + const double xi_231 = xi_103*-0.0238095238095238 + xi_127*-0.00396825396825397; + const double xi_236 = xi_146*xi_226; + const double xi_237 = xi_135*xi_228; + const double xi_241 = -xi_193; + const double xi_244 = xi_107*0.0357142857142857; + const double xi_246 = xi_153*(random_0_3 - 0.5); + const double xi_251 = xi_170*xi_228; + const double xi_252 = xi_174*xi_226; + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_41 = u_0*xi_268; + const double xi_42 = xi_41*0.333333333333333; + const double xi_48 = -xi_42; + const double xi_114 = rho*(u_0*u_0); + const double xi_165 = rho*u_0; + const double xi_166 = -vel0Term + xi_165 + xi_32; + const double xi_167 = xi_133*xi_166; + const double xi_202 = xi_166*xi_197; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_43 = u_1*xi_272; + const double xi_44 = xi_43*0.333333333333333; + const double xi_49 = -xi_44; + const double xi_71 = u_1*0.5; + const double xi_74 = xi_73*(u_0*xi_72 + xi_268*xi_71); + const double xi_75 = -xi_74; + const double xi_119 = rho*(u_1*u_1); + const double xi_120 = xi_118 + xi_119 + xi_20; + const double xi_131 = rho*u_1; + const double xi_132 = -vel1Term + xi_118 + xi_131 + xi_273 + xi_37; + const double xi_134 = xi_132*xi_133; + const double xi_198 = xi_132*xi_197; + const double u_2 = xi_11*(vel2Term + xi_260 + xi_27 + xi_30); + const double xi_45 = u_2*xi_275; + const double xi_46 = xi_45*0.333333333333333; + const double xi_47 = (xi_40 + 2.0)*(xi_42 + xi_44 + xi_46); + const double xi_50 = xi_45*0.666666666666667 + xi_48 + xi_49; + const double xi_54 = -xi_46; + const double xi_55 = xi_43*0.666666666666667 + xi_48 + xi_54; + const double xi_56 = xi_41*0.666666666666667 + xi_49 + xi_54; + const double xi_59 = xi_50*xi_58; + const double xi_60 = -xi_59; + const double xi_61 = xi_56*xi_58; + const double xi_62 = -xi_61; + const double xi_64 = xi_55*xi_63 + xi_60 + xi_62; + const double xi_66 = xi_55*xi_58; + const double xi_67 = -xi_66; + const double xi_68 = xi_56*xi_63 + xi_60 + xi_67; + const double xi_70 = xi_50*xi_63 + xi_62 + xi_67; + const double xi_77 = xi_61 - xi_76; + const double xi_79 = -xi_50*xi_78; + const double xi_81 = xi_47*0.125; + const double xi_82 = xi_66 + xi_81; + const double xi_83 = xi_80 + xi_82; + const double xi_84 = xi_79 + xi_83; + const double xi_85 = xi_61 + xi_76; + const double xi_86 = -xi_80 + xi_82; + const double xi_87 = xi_79 + xi_86; + const double xi_88 = xi_73*(u_2*xi_72 + xi_275*xi_71); + const double xi_89 = -xi_56*xi_78; + const double xi_91 = xi_59 + xi_90; + const double xi_92 = xi_89 + xi_91; + const double xi_93 = -xi_88; + const double xi_94 = xi_73*(u_0*xi_275*0.5 + u_2*xi_268*0.5); + const double xi_95 = -xi_94; + const double xi_96 = -xi_55*xi_78; + const double xi_97 = xi_81 + xi_91 + xi_96; + const double xi_98 = xi_59 - xi_90; + const double xi_99 = xi_89 + xi_98; + const double xi_100 = xi_81 + xi_96 + xi_98; + const double xi_115 = rho*(u_2*u_2); + const double xi_123 = omega_bulk*(xi_114 + xi_115 + xi_117 + xi_120 + xi_122 + xi_24 + xi_271 + xi_28); + const double xi_155 = -xi_115 + xi_264 + xi_274; + const double xi_156 = omega_shear*(xi_1 + xi_120 + xi_155 + xi_18 + xi_23 - xi_276 + xi_31); + const double xi_157 = xi_156*0.125; + const double xi_159 = omega_shear*(xi_111 + xi_114*2.0 + xi_118 - xi_119 + xi_122 + xi_155 + xi_20 + xi_259 + xi_261*-2.0 + xi_276 + xi_279*-2.0); + const double xi_161 = xi_159*-0.0416666666666667 + xi_160*-0.166666666666667; + const double xi_162 = xi_104*-0.1 + xi_110*-0.05 + xi_161; + const double xi_163 = xi_107*0.0285714285714286 + xi_113*0.0142857142857143 + xi_154 + xi_157 + xi_158 + xi_162; + const double xi_177 = xi_107*-0.0714285714285714 + xi_113*-0.0357142857142857 + xi_158 + xi_159*0.0833333333333333 + xi_160*0.333333333333333; + const double xi_182 = rho*u_2 - vel2Term + xi_10 + xi_116 + xi_121 + xi_179 + xi_266; + const double xi_183 = xi_133*xi_182; + const double xi_191 = xi_103*0.0952380952380952 + xi_107*-0.0428571428571429 + xi_113*-0.0214285714285714 + xi_127*0.0158730158730159 - xi_154 - xi_157 + xi_162; + const double xi_194 = xi_156*0.0625; + const double xi_199 = xi_106*0.0833333333333333 + xi_123*0.0416666666666667; + const double xi_200 = xi_198 + xi_199; + const double xi_201 = xi_164 + xi_193 + xi_194 + xi_195 + xi_196 + xi_200; + const double xi_203 = xi_159*0.0208333333333333 + xi_160*0.0833333333333333; + const double xi_204 = -xi_202 + xi_203; + const double xi_205 = xi_178 + xi_204; + const double xi_211 = xi_202 + xi_203; + const double xi_212 = xi_176 + xi_211; + const double xi_213 = -xi_198 + xi_199; + const double xi_214 = xi_150 + xi_193 + xi_194 + xi_195 + xi_196 + xi_213; + const double xi_230 = xi_182*xi_197; + const double xi_232 = xi_230 + xi_231; + const double xi_233 = -xi_223 + xi_225 - xi_227 + xi_229 + xi_232; + const double xi_238 = xi_200 - xi_234 + xi_235 - xi_236 + xi_237; + const double xi_239 = xi_213 + xi_234 - xi_235 + xi_236 - xi_237; + const double xi_242 = -xi_194; + const double xi_245 = xi_192 + xi_199 + xi_232 + xi_241 + xi_242 + xi_243 + xi_244; + const double xi_253 = xi_204 - xi_249 + xi_250 - xi_251 + xi_252; + const double xi_255 = xi_211 + xi_249 - xi_250 + xi_251 - xi_252; + const double xi_256 = -xi_230 + xi_231; + const double xi_257 = xi_223 - xi_225 + xi_227 - xi_229 + xi_256; + const double xi_258 = xi_190 + xi_199 + xi_241 + xi_242 + xi_243 + xi_244 + xi_256; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_208 = xi_207*(-p_1 + u_0*xi_131); + const double xi_209 = -xi_206 - xi_208; + const double xi_210 = xi_206 + xi_208; + const double p_2 = xi_36; + const double xi_247 = xi_207*(-p_2 + u_2*xi_165); + const double xi_248 = -xi_246 - xi_247; + const double xi_254 = xi_246 + xi_247; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_217 = xi_207*(-p_5 + u_2*xi_131); + const double xi_221 = xi_161 + xi_215 + xi_216 + xi_217 + xi_218 + xi_219 + xi_220; + const double xi_240 = xi_161 - xi_215 + xi_216 - xi_217 + xi_218 + xi_219 + xi_220; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_262 + xi_3 + xi_7; + const double forceTerm_0 = xi_47*-1.5 - xi_50*xi_53 - xi_53*xi_55 - xi_53*xi_56; + const double forceTerm_1 = xi_57 + xi_64; + const double forceTerm_2 = -xi_57 + xi_64; + const double forceTerm_3 = -xi_65 + xi_68; + const double forceTerm_4 = xi_65 + xi_68; + const double forceTerm_5 = xi_69 + xi_70; + const double forceTerm_6 = -xi_69 + xi_70; + const double forceTerm_7 = xi_75 + xi_77 + xi_84; + const double forceTerm_8 = xi_74 + xi_84 + xi_85; + const double forceTerm_9 = xi_74 + xi_77 + xi_87; + const double forceTerm_10 = xi_75 + xi_85 + xi_87; + const double forceTerm_11 = xi_83 + xi_88 + xi_92; + const double forceTerm_12 = xi_86 + xi_92 + xi_93; + const double forceTerm_13 = xi_77 + xi_95 + xi_97; + const double forceTerm_14 = xi_85 + xi_94 + xi_97; + const double forceTerm_15 = xi_83 + xi_93 + xi_99; + const double forceTerm_16 = xi_86 + xi_88 + xi_99; + const double forceTerm_17 = xi_100 + xi_77 + xi_94; + const double forceTerm_18 = xi_100 + xi_85 + xi_95; + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0] = forceTerm_0 + xi_103*0.142857142857143 + xi_104*0.2 - xi_106 + xi_107*0.0857142857142857 + xi_110*0.1 + xi_113*0.0428571428571429 + xi_123*-0.5 + xi_127*0.0238095238095238 + xi_271; + _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0] = forceTerm_1 - xi_130 + xi_134 - xi_139 + xi_150 + xi_163 + xi_276; + _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0] = forceTerm_2 + xi_130 - xi_134 + xi_139 + xi_163 + xi_164 + xi_259; + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0] = forceTerm_3 - xi_167 + xi_169 + xi_171 + xi_176 + xi_177 + xi_279; + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0] = forceTerm_4 + xi_167 - xi_169 - xi_171 + xi_177 + xi_178 + xi_261; + _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0] = forceTerm_5 - xi_181 + xi_183 - xi_185 + xi_190 + xi_191 + xi_274; + _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0] = forceTerm_6 + xi_181 - xi_183 + xi_185 + xi_191 + xi_192 + xi_264; + _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0] = forceTerm_7 + xi_201 + xi_205 + xi_209 + xi_280; + _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0] = forceTerm_8 + xi_201 + xi_210 + xi_212 + xi_263; + _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0] = forceTerm_9 + xi_205 + xi_210 + xi_214 + xi_269; + _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0] = forceTerm_10 + xi_209 + xi_212 + xi_214 + xi_273; + _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0] = forceTerm_11 + xi_221 + xi_233 + xi_238 + xi_265; + _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0] = forceTerm_12 + xi_233 + xi_239 + xi_240 + xi_277; + _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0] = forceTerm_13 + xi_245 + xi_248 + xi_253 + xi_267; + _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0] = forceTerm_14 + xi_245 + xi_254 + xi_255 + xi_260; + _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0] = forceTerm_15 + xi_238 + xi_240 + xi_257 + xi_270; + _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0] = forceTerm_16 + xi_221 + xi_239 + xi_257 + xi_262; + _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0] = forceTerm_17 + xi_253 + xi_254 + xi_258 + xi_278; + _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0] = forceTerm_18 + xi_248 + xi_255 + xi_258 + xi_266; + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void kernel_stream( - double *RESTRICT const _data_pdfs, double *RESTRICT _data_pdfs_tmp, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) { - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) + { + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } } -} // namespace internal_kernel_stream +} + -const real_t FluctuatingMRTLatticeModel::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t FluctuatingMRTLatticeModel::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; +const real_t FluctuatingMRTLatticeModel::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t FluctuatingMRTLatticeModel::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; -void FluctuatingMRTLatticeModel::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +void FluctuatingMRTLatticeModel::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &seed = lm.seed_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - auto &block_offset_2 = lm.block_offset_2_; - auto &block_offset_0 = lm.block_offset_0_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, - _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, - _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, - block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, - time_step); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_even = lm.omega_even_; + auto & block_offset_1 = lm.block_offset_1_; + auto & seed = lm.seed_; + auto & time_step = lm.time_step_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & block_offset_2 = lm.block_offset_2_; + auto & omega_odd = lm.omega_odd_; + auto & kT = lm.kT_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); + pdfs->swapDataPointers(pdfs_tmp); + } -void FluctuatingMRTLatticeModel::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void FluctuatingMRTLatticeModel::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &seed = lm.seed_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - auto &block_offset_2 = lm.block_offset_2_; - auto &block_offset_0 = lm.block_offset_0_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, - omega_even, omega_odd, omega_shear, seed, time_step); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_even = lm.omega_even_; + auto & block_offset_1 = lm.block_offset_1_; + auto & seed = lm.seed_; + auto & time_step = lm.time_step_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & block_offset_2 = lm.block_offset_2_; + auto & omega_odd = lm.omega_odd_; + auto & kT = lm.kT_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); } -void FluctuatingMRTLatticeModel::Sweep::stream( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void FluctuatingMRTLatticeModel::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, - _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer & -operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::FluctuatingMRTLatticeModel &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::FluctuatingMRTLatticeModel & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer &operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::FluctuatingMRTLatticeModel &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::FluctuatingMRTLatticeModel & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h index 8b510541764..ab6ab7be70f 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModel.h @@ -696,16 +696,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModel & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const FluctuatingMRTLatticeModel & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModel & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const FluctuatingMRTLatticeModel & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp index 17d02bbf26d..c78023230f5 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.cpp @@ -18,1752 +18,1185 @@ #include -#include "FluctuatingMRTLatticeModelAvx.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "FluctuatingMRTLatticeModelAvx.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + #include "philox_rand.h" + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_0, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, - uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, - double kT, double omega_bulk, double omega_even, double omega_odd, - double omega_shear, uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) { - - float random_3_0; - float random_3_1; - float random_3_2; - float random_3_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, - random_3_2, random_3_3); - - float random_2_0; - float random_2_1; - float random_2_2; - float random_2_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, - random_2_2, random_2_3); - - float random_1_0; - float random_1_1; - float random_1_2; - float random_1_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, - random_1_2, random_1_3); - - float random_0_0; - float random_0_1; - float random_0_2; - float random_0_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, - random_0_2, random_0_3); - - const double xi_0 = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_1 = - xi_0 + _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_2 = _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_3 = _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_4 = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_5 = _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_6 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_8 = - -_data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_9 = - xi_8 - - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_10 = - -_data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_11 = - -_data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_12 = - -_data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -_data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - const double xi_15 = - -_data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -_data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_18 = -_data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = - -_data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -_data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_23 = -_data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_24 = xi_17 + xi_22 + xi_23 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_42 = - 0.166666666666667 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_50 = - 0.166666666666667 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_54 = - 0.166666666666667 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_57 = - 0.5 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_61 = - 0.0833333333333333 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_65 = - 0.0833333333333333 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_75 = - 0.0833333333333333 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_93 = -_data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_94 = xi_93 + - 3.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_95 = - omega_even * - (xi_94 - 3.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_96 = - 2.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_97 = - xi_96 + - 5.0 * _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_98 = - omega_even * - (xi_94 + xi_97 - - 2.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] - - 2.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] - - 5.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 5.0 * - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - - _stride_pdfs_0]); - const double xi_101 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_102 = xi_101 + xi_18; - const double xi_103 = - -_data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_106 = - -_data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_107 = xi_106 + xi_11 + xi_15 + xi_21; - const double xi_109 = - 2.0 * - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_110 = - 2.0 * - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_111 = - 2.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 2.0 * _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_112 = - omega_even * - (xi_109 + xi_110 + xi_111 + xi_93 + xi_97 - - 4.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] - - 4.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0] - - 7.0 * - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 5.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_113 = - xi_101 + _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_114 = xi_113 + xi_14 + xi_22 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_116 = xi_114 * xi_115; - const double xi_118 = - xi_103 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_122 = random_2_3 - 0.5; - const double xi_127 = - 2.0 * _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_128 = - 2.0 * _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_129 = - -2.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - 2.0 * _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_130 = -xi_127 + xi_128 + xi_129 + xi_14 + xi_19 + xi_2; - const double xi_132 = xi_130 * xi_131; - const double xi_133 = random_1_2 - 0.5; - const double xi_138 = random_0_1 - 0.5; - const double xi_142 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_156 = - xi_106 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_157 = - xi_12 + xi_156 + xi_20 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_158 = xi_115 * xi_157; - const double xi_159 = random_2_1 - 0.5; - const double xi_161 = xi_1 + xi_127 - xi_128 + xi_129 + xi_13; - const double xi_162 = xi_131 * xi_161; - const double xi_163 = random_2_0 - 0.5; - const double xi_168 = _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_169 = xi_102 + xi_168 + xi_23 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_170 = xi_115 * xi_169; - const double xi_173 = random_2_2 - 0.5; - const double xi_175 = -xi_109 - xi_110 + xi_111 + xi_24 + xi_3; - const double xi_176 = xi_131 * xi_175; - const double xi_177 = random_1_3 - 0.5; - const double xi_184 = xi_112 * 0.0138888888888889; - const double xi_205 = xi_98 * -0.00714285714285714; - const double xi_207 = xi_95 * 0.025; - const double xi_212 = xi_175 * xi_211; - const double xi_214 = xi_169 * xi_213; - const double xi_223 = xi_130 * xi_211; - const double xi_224 = xi_114 * xi_213; - const double xi_232 = xi_98 * 0.0178571428571429; - const double xi_238 = xi_157 * xi_213; - const double xi_239 = xi_161 * xi_211; - const double vel0Term = - xi_1 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double vel1Term = - xi_2 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double vel2Term = - xi_3 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double rho = vel0Term + vel1Term + vel2Term + xi_4 + xi_5 + xi_6 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_7 = 1 / (rho); - const double xi_86 = kT * rho; - const double xi_87 = - sqrt(xi_86 * (-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0)); - const double xi_88 = xi_87 * (random_3_0 - 0.5) * 3.7416573867739413; - const double xi_89 = xi_87 * (random_3_2 - 0.5) * 5.4772255750516612; - const double xi_91 = - xi_90 * sqrt(xi_86 * (-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0)) * - (random_1_1 - 0.5); - const double xi_92 = xi_87 * (random_3_1 - 0.5) * 8.3666002653407556; - const double xi_123 = - sqrt(xi_86 * (-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0)); - const double xi_124 = xi_123 * 1.4142135623730951; - const double xi_125 = xi_124 * 0.5; - const double xi_126 = xi_122 * xi_125; - const double xi_134 = xi_123 * xi_90; - const double xi_135 = xi_134 * 0.166666666666667; - const double xi_136 = xi_133 * xi_135; - const double xi_137 = -xi_132 - xi_136; - const double xi_139 = - sqrt(xi_86 * (-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0)); - const double xi_140 = xi_139 * 0.5; - const double xi_141 = xi_138 * xi_140; - const double xi_146 = - xi_112 * -0.0198412698412698 + xi_88 * -0.119047619047619; - const double xi_148 = xi_139 * (random_0_0 - 0.5) * 1.7320508075688772; - const double xi_152 = xi_132 + xi_136; - const double xi_160 = xi_125 * xi_159; - const double xi_164 = xi_135 * xi_163; - const double xi_165 = xi_162 + xi_164; - const double xi_167 = -xi_162 - xi_164; - const double xi_174 = xi_125 * xi_173; - const double xi_178 = xi_135 * xi_177; - const double xi_179 = -xi_176 - xi_178; - const double xi_181 = xi_176 + xi_178; - const double xi_182 = xi_138 * xi_139 * 0.25; - const double xi_185 = xi_88 * 0.0833333333333333; - const double xi_195 = xi_140 * (random_0_2 - 0.5); - const double xi_204 = xi_140 * (random_1_0 - 0.5); - const double xi_208 = xi_92 * -0.0142857142857143; - const double xi_209 = xi_89 * 0.05; - const double xi_215 = xi_134 * 0.0833333333333333; - const double xi_216 = xi_177 * xi_215; - const double xi_217 = xi_124 * 0.25; - const double xi_218 = xi_173 * xi_217; - const double xi_220 = - xi_112 * -0.00396825396825397 + xi_88 * -0.0238095238095238; - const double xi_225 = xi_133 * xi_215; - const double xi_226 = xi_122 * xi_217; - const double xi_230 = -xi_182; - const double xi_233 = xi_92 * 0.0357142857142857; - const double xi_235 = xi_140 * (random_0_3 - 0.5); - const double xi_240 = xi_159 * xi_217; - const double xi_241 = xi_163 * xi_215; - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_26 = - u_0 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_27 = xi_26 * 0.333333333333333; - const double xi_33 = -xi_27; - const double xi_99 = rho * (u_0 * u_0); - const double xi_153 = rho * u_0; - const double xi_154 = - -vel0Term + xi_142 + xi_153 + xi_4 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_155 = xi_120 * xi_154; - const double xi_191 = xi_154 * xi_186; - const double u_1 = - xi_7 * - (vel1Term + xi_16 + xi_19 + xi_8 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_28 = - u_1 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_29 = xi_28 * 0.333333333333333; - const double xi_34 = -xi_29; - const double xi_56 = u_1 * 0.5; - const double xi_59 = - xi_58 * (u_0 * xi_57 + - xi_56 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_60 = -xi_59; - const double xi_104 = rho * (u_1 * u_1); - const double xi_105 = xi_103 + xi_104 + xi_9; - const double xi_117 = rho * u_1; - const double xi_119 = - -vel1Term + xi_117 + xi_118 + xi_5 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_121 = xi_119 * xi_120; - const double xi_187 = xi_119 * xi_186; - const double xi_197 = - xi_196 * - (u_0 * xi_117 + xi_118 + xi_8 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]); - const double xi_198 = -xi_195 - xi_197; - const double xi_199 = xi_195 + xi_197; - const double u_2 = - xi_7 * - (vel2Term + xi_21 + xi_24 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_30 = - u_2 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_31 = xi_30 * 0.333333333333333; - const double xi_32 = (xi_25 + 2.0) * (xi_27 + xi_29 + xi_31); - const double xi_35 = xi_30 * 0.666666666666667 + xi_33 + xi_34; - const double xi_39 = -xi_31; - const double xi_40 = xi_28 * 0.666666666666667 + xi_33 + xi_39; - const double xi_41 = xi_26 * 0.666666666666667 + xi_34 + xi_39; - const double xi_44 = xi_35 * xi_43; - const double xi_45 = -xi_44; - const double xi_46 = xi_41 * xi_43; - const double xi_47 = -xi_46; - const double xi_49 = xi_40 * xi_48 + xi_45 + xi_47; - const double xi_51 = xi_40 * xi_43; - const double xi_52 = -xi_51; - const double xi_53 = xi_41 * xi_48 + xi_45 + xi_52; - const double xi_55 = xi_35 * xi_48 + xi_47 + xi_52; - const double xi_62 = xi_46 - xi_61; - const double xi_64 = -xi_35 * xi_63; - const double xi_66 = xi_32 * 0.125; - const double xi_67 = xi_51 + xi_66; - const double xi_68 = xi_65 + xi_67; - const double xi_69 = xi_64 + xi_68; - const double xi_70 = xi_46 + xi_61; - const double xi_71 = -xi_65 + xi_67; - const double xi_72 = xi_64 + xi_71; - const double xi_73 = - xi_58 * (u_2 * xi_57 + - xi_56 * _data_force_20_32_10[_stride_force_0 * ctr_0]); - const double xi_74 = -xi_41 * xi_63; - const double xi_76 = xi_44 + xi_75; - const double xi_77 = xi_74 + xi_76; - const double xi_78 = -xi_73; - const double xi_79 = - xi_58 * (u_0 * 0.5 * _data_force_20_32_10[_stride_force_0 * ctr_0] + - u_2 * 0.5 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_80 = -xi_79; - const double xi_81 = -xi_40 * xi_63; - const double xi_82 = xi_66 + xi_76 + xi_81; - const double xi_83 = xi_44 - xi_75; - const double xi_84 = xi_74 + xi_83; - const double xi_85 = xi_66 + xi_81 + xi_83; - const double xi_100 = rho * (u_2 * u_2); - const double xi_108 = - omega_bulk * (xi_100 + xi_102 + xi_105 + xi_107 + xi_17 + xi_22 + - xi_99 + _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]); - const double xi_143 = -xi_100 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_144 = - omega_shear * (xi_0 + xi_105 + xi_142 + xi_143 + xi_16 - - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_145 = xi_144 * 0.125; - const double xi_147 = - omega_shear * - (xi_103 - xi_104 + xi_107 + xi_143 + xi_9 + xi_96 + xi_99 * 2.0 - - 2.0 * - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 2.0 * - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_149 = - xi_147 * -0.0416666666666667 + xi_148 * -0.166666666666667; - const double xi_150 = xi_149 + xi_89 * -0.1 + xi_95 * -0.05; - const double xi_151 = xi_141 + xi_145 + xi_146 + xi_150 + - xi_92 * 0.0285714285714286 + - xi_98 * 0.0142857142857143; - const double xi_166 = - xi_146 + xi_147 * 0.0833333333333333 + xi_148 * 0.333333333333333 + - xi_92 * -0.0714285714285714 + xi_98 * -0.0357142857142857; - const double xi_171 = - rho * u_2 - vel2Term + xi_101 + xi_106 + xi_168 + xi_6 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_172 = xi_120 * xi_171; - const double xi_180 = xi_112 * 0.0158730158730159 - xi_141 - xi_145 + - xi_150 + xi_88 * 0.0952380952380952 + - xi_92 * -0.0428571428571429 + - xi_98 * -0.0214285714285714; - const double xi_183 = xi_144 * 0.0625; - const double xi_188 = - xi_108 * 0.0416666666666667 + xi_91 * 0.0833333333333333; - const double xi_189 = xi_187 + xi_188; - const double xi_190 = - xi_152 + xi_182 + xi_183 + xi_184 + xi_185 + xi_189; - const double xi_192 = - xi_147 * 0.0208333333333333 + xi_148 * 0.0833333333333333; - const double xi_193 = -xi_191 + xi_192; - const double xi_194 = xi_167 + xi_193; - const double xi_200 = xi_191 + xi_192; - const double xi_201 = xi_165 + xi_200; - const double xi_202 = -xi_187 + xi_188; - const double xi_203 = - xi_137 + xi_182 + xi_183 + xi_184 + xi_185 + xi_202; - const double xi_206 = - xi_196 * (u_2 * xi_117 + xi_113 + xi_17 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_210 = - xi_149 + xi_204 + xi_205 + xi_206 + xi_207 + xi_208 + xi_209; - const double xi_219 = xi_171 * xi_186; - const double xi_221 = xi_219 + xi_220; - const double xi_222 = -xi_212 + xi_214 - xi_216 + xi_218 + xi_221; - const double xi_227 = xi_189 - xi_223 + xi_224 - xi_225 + xi_226; - const double xi_228 = xi_202 + xi_223 - xi_224 + xi_225 - xi_226; - const double xi_229 = - xi_149 - xi_204 + xi_205 - xi_206 + xi_207 + xi_208 + xi_209; - const double xi_231 = -xi_183; - const double xi_234 = - xi_181 + xi_188 + xi_221 + xi_230 + xi_231 + xi_232 + xi_233; - const double xi_236 = - xi_196 * - (u_2 * xi_153 + xi_10 + xi_156 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_237 = -xi_235 - xi_236; - const double xi_242 = xi_193 - xi_238 + xi_239 - xi_240 + xi_241; - const double xi_243 = xi_235 + xi_236; - const double xi_244 = xi_200 + xi_238 - xi_239 + xi_240 - xi_241; - const double xi_245 = -xi_219 + xi_220; - const double xi_246 = xi_212 - xi_214 + xi_216 - xi_218 + xi_245; - const double xi_247 = - xi_179 + xi_188 + xi_230 + xi_231 + xi_232 + xi_233 + xi_245; - const double forceTerm_0 = - xi_32 * -1.5 - xi_35 * xi_38 - xi_38 * xi_40 - xi_38 * xi_41; - const double forceTerm_1 = xi_42 + xi_49; - const double forceTerm_2 = -xi_42 + xi_49; - const double forceTerm_3 = -xi_50 + xi_53; - const double forceTerm_4 = xi_50 + xi_53; - const double forceTerm_5 = xi_54 + xi_55; - const double forceTerm_6 = -xi_54 + xi_55; - const double forceTerm_7 = xi_60 + xi_62 + xi_69; - const double forceTerm_8 = xi_59 + xi_69 + xi_70; - const double forceTerm_9 = xi_59 + xi_62 + xi_72; - const double forceTerm_10 = xi_60 + xi_70 + xi_72; - const double forceTerm_11 = xi_68 + xi_73 + xi_77; - const double forceTerm_12 = xi_71 + xi_77 + xi_78; - const double forceTerm_13 = xi_62 + xi_80 + xi_82; - const double forceTerm_14 = xi_70 + xi_79 + xi_82; - const double forceTerm_15 = xi_68 + xi_78 + xi_84; - const double forceTerm_16 = xi_71 + xi_73 + xi_84; - const double forceTerm_17 = xi_62 + xi_79 + xi_85; - const double forceTerm_18 = xi_70 + xi_80 + xi_85; - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_0 + xi_108 * -0.5 + xi_112 * 0.0238095238095238 + - xi_88 * 0.142857142857143 + xi_89 * 0.2 - xi_91 + - xi_92 * 0.0857142857142857 + xi_95 * 0.1 + - xi_98 * 0.0428571428571429 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_1 - xi_116 + xi_121 - xi_126 + xi_137 + xi_151 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_2 + xi_116 - xi_121 + xi_126 + xi_151 + xi_152 + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_3 - xi_155 + xi_158 + xi_160 + xi_165 + xi_166 + - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_4 + xi_155 - xi_158 - xi_160 + xi_166 + xi_167 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_5 - xi_170 + xi_172 - xi_174 + xi_179 + xi_180 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_6 + xi_170 - xi_172 + xi_174 + xi_180 + xi_181 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_7 + xi_190 + xi_194 + xi_198 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_8 + xi_190 + xi_199 + xi_201 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_9 + xi_194 + xi_199 + xi_203 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_10 + xi_198 + xi_201 + xi_203 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_11 + xi_210 + xi_222 + xi_227 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_12 + xi_222 + xi_228 + xi_229 + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_13 + xi_234 + xi_237 + xi_242 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_14 + xi_234 + xi_243 + xi_244 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_15 + xi_227 + xi_229 + xi_246 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_16 + xi_210 + xi_228 + xi_246 + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_17 + xi_242 + xi_243 + xi_247 + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_18 + xi_237 + xi_244 + xi_247 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) + { + + float random_3_0; + float random_3_1; + float random_3_2; + float random_3_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, random_3_2, random_3_3); + + + float random_2_0; + float random_2_1; + float random_2_2; + float random_2_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, random_2_2, random_2_3); + + + float random_1_0; + float random_1_1; + float random_1_2; + float random_1_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, random_1_2, random_1_3); + + + float random_0_0; + float random_0_1; + float random_0_2; + float random_0_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, random_0_2, random_0_3); + + const double xi_0 = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_1 = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_2 = xi_0 + xi_1 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_3 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_4 = xi_3 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + const double xi_5 = xi_4 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_6 = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_7 = xi_6 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_8 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_9 = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_10 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_12 = -_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_13 = -_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_16 = -_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_17 = -_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_21 = xi_20 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_22 = -_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_23 = -_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + const double xi_24 = -_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + const double xi_29 = -_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_30 = xi_24 + xi_28 + xi_29 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_31 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_32 = xi_31 + xi_8 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_33 = xi_16 + xi_21 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_34 = xi_26 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_35 = xi_15 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_9 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_38 = xi_28 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_39 = xi_22 + xi_38 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_57 = 0.166666666666667*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_65 = 0.166666666666667*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_69 = 0.166666666666667*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_72 = 0.5*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_76 = 0.0833333333333333*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_80 = 0.0833333333333333*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_90 = 0.0833333333333333*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_108 = -_data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_109 = xi_108 + 3.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_110 = omega_even*(xi_109 - 3.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_111 = 2.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_112 = xi_111 + 5.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_113 = omega_even*(xi_109 + xi_112 - 2.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] - 2.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] - 5.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 5.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_116 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_117 = xi_116 + xi_22; + const double xi_118 = -_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_121 = -_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_122 = xi_121 + xi_19 + xi_27; + const double xi_124 = 2.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_125 = 2.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_126 = 2.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 2.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_127 = omega_even*(xi_108 + xi_112 + xi_124 + xi_125 + xi_126 - 4.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] - 4.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0] - 7.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 7.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 5.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_128 = xi_116 + xi_23 + xi_38 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_130 = xi_128*xi_129; + const double xi_135 = random_2_3 - 0.5; + const double xi_140 = 2.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_141 = 2.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_142 = -2.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + 2.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_143 = -xi_140 + xi_141 + xi_142 + xi_25 + xi_4; + const double xi_145 = xi_143*xi_144; + const double xi_146 = random_1_2 - 0.5; + const double xi_151 = random_0_1 - 0.5; + const double xi_168 = xi_121 + xi_13 + xi_34 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_169 = xi_129*xi_168; + const double xi_170 = random_2_1 - 0.5; + const double xi_172 = xi_14 + xi_140 - xi_141 + xi_142 + xi_35 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_173 = xi_144*xi_172; + const double xi_174 = random_2_0 - 0.5; + const double xi_179 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_180 = xi_117 + xi_179 + xi_29 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_181 = xi_129*xi_180; + const double xi_184 = random_2_2 - 0.5; + const double xi_186 = -xi_124 - xi_125 + xi_126 + xi_30 + xi_6; + const double xi_187 = xi_144*xi_186; + const double xi_188 = random_1_3 - 0.5; + const double xi_195 = xi_127*0.0138888888888889; + const double xi_216 = xi_113*-0.00714285714285714; + const double xi_218 = xi_110*0.025; + const double xi_223 = xi_186*xi_222; + const double xi_225 = xi_180*xi_224; + const double xi_234 = xi_143*xi_222; + const double xi_235 = xi_128*xi_224; + const double xi_243 = xi_113*0.0178571428571429; + const double xi_249 = xi_168*xi_224; + const double xi_250 = xi_172*xi_222; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_8 + xi_9 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_11 = 1 / (rho); + const double xi_101 = kT*rho; + const double xi_102 = sqrt(xi_101*(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0)); + const double xi_103 = xi_102*(random_3_0 - 0.5)*3.7416573867739413; + const double xi_104 = xi_102*(random_3_2 - 0.5)*5.4772255750516612; + const double xi_106 = xi_105*sqrt(xi_101*(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))*(random_1_1 - 0.5); + const double xi_107 = xi_102*(random_3_1 - 0.5)*8.3666002653407556; + const double xi_136 = sqrt(xi_101*(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0)); + const double xi_137 = xi_136*1.4142135623730951; + const double xi_138 = xi_137*0.5; + const double xi_139 = xi_135*xi_138; + const double xi_147 = xi_105*xi_136; + const double xi_148 = xi_147*0.166666666666667; + const double xi_149 = xi_146*xi_148; + const double xi_150 = -xi_145 - xi_149; + const double xi_152 = sqrt(xi_101*(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0)); + const double xi_153 = xi_152*0.5; + const double xi_154 = xi_151*xi_153; + const double xi_158 = xi_103*-0.119047619047619 + xi_127*-0.0198412698412698; + const double xi_160 = xi_152*(random_0_0 - 0.5)*1.7320508075688772; + const double xi_164 = xi_145 + xi_149; + const double xi_171 = xi_138*xi_170; + const double xi_175 = xi_148*xi_174; + const double xi_176 = xi_173 + xi_175; + const double xi_178 = -xi_173 - xi_175; + const double xi_185 = xi_138*xi_184; + const double xi_189 = xi_148*xi_188; + const double xi_190 = -xi_187 - xi_189; + const double xi_192 = xi_187 + xi_189; + const double xi_193 = xi_151*xi_152*0.25; + const double xi_196 = xi_103*0.0833333333333333; + const double xi_206 = xi_153*(random_0_2 - 0.5); + const double xi_215 = xi_153*(random_1_0 - 0.5); + const double xi_219 = xi_107*-0.0142857142857143; + const double xi_220 = xi_104*0.05; + const double xi_226 = xi_147*0.0833333333333333; + const double xi_227 = xi_188*xi_226; + const double xi_228 = xi_137*0.25; + const double xi_229 = xi_184*xi_228; + const double xi_231 = xi_103*-0.0238095238095238 + xi_127*-0.00396825396825397; + const double xi_236 = xi_146*xi_226; + const double xi_237 = xi_135*xi_228; + const double xi_241 = -xi_193; + const double xi_244 = xi_107*0.0357142857142857; + const double xi_246 = xi_153*(random_0_3 - 0.5); + const double xi_251 = xi_170*xi_228; + const double xi_252 = xi_174*xi_226; + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_41 = u_0*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_42 = xi_41*0.333333333333333; + const double xi_48 = -xi_42; + const double xi_114 = rho*(u_0*u_0); + const double xi_165 = rho*u_0; + const double xi_166 = -vel0Term + xi_165 + xi_32; + const double xi_167 = xi_133*xi_166; + const double xi_202 = xi_166*xi_197; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_43 = u_1*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_44 = xi_43*0.333333333333333; + const double xi_49 = -xi_44; + const double xi_71 = u_1*0.5; + const double xi_74 = xi_73*(u_0*xi_72 + xi_71*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_75 = -xi_74; + const double xi_119 = rho*(u_1*u_1); + const double xi_120 = xi_118 + xi_119 + xi_20; + const double xi_131 = rho*u_1; + const double xi_132 = -vel1Term + xi_118 + xi_131 + xi_37 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_134 = xi_132*xi_133; + const double xi_198 = xi_132*xi_197; + const double u_2 = xi_11*(vel2Term + xi_27 + xi_30 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_45 = u_2*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_46 = xi_45*0.333333333333333; + const double xi_47 = (xi_40 + 2.0)*(xi_42 + xi_44 + xi_46); + const double xi_50 = xi_45*0.666666666666667 + xi_48 + xi_49; + const double xi_54 = -xi_46; + const double xi_55 = xi_43*0.666666666666667 + xi_48 + xi_54; + const double xi_56 = xi_41*0.666666666666667 + xi_49 + xi_54; + const double xi_59 = xi_50*xi_58; + const double xi_60 = -xi_59; + const double xi_61 = xi_56*xi_58; + const double xi_62 = -xi_61; + const double xi_64 = xi_55*xi_63 + xi_60 + xi_62; + const double xi_66 = xi_55*xi_58; + const double xi_67 = -xi_66; + const double xi_68 = xi_56*xi_63 + xi_60 + xi_67; + const double xi_70 = xi_50*xi_63 + xi_62 + xi_67; + const double xi_77 = xi_61 - xi_76; + const double xi_79 = -xi_50*xi_78; + const double xi_81 = xi_47*0.125; + const double xi_82 = xi_66 + xi_81; + const double xi_83 = xi_80 + xi_82; + const double xi_84 = xi_79 + xi_83; + const double xi_85 = xi_61 + xi_76; + const double xi_86 = -xi_80 + xi_82; + const double xi_87 = xi_79 + xi_86; + const double xi_88 = xi_73*(u_2*xi_72 + xi_71*_data_force_20_32_10[_stride_force_0*ctr_0]); + const double xi_89 = -xi_56*xi_78; + const double xi_91 = xi_59 + xi_90; + const double xi_92 = xi_89 + xi_91; + const double xi_93 = -xi_88; + const double xi_94 = xi_73*(u_0*0.5*_data_force_20_32_10[_stride_force_0*ctr_0] + u_2*0.5*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_95 = -xi_94; + const double xi_96 = -xi_55*xi_78; + const double xi_97 = xi_81 + xi_91 + xi_96; + const double xi_98 = xi_59 - xi_90; + const double xi_99 = xi_89 + xi_98; + const double xi_100 = xi_81 + xi_96 + xi_98; + const double xi_115 = rho*(u_2*u_2); + const double xi_123 = omega_bulk*(xi_114 + xi_115 + xi_117 + xi_120 + xi_122 + xi_24 + xi_28 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]); + const double xi_155 = -xi_115 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_156 = omega_shear*(xi_1 + xi_120 + xi_155 + xi_18 + xi_23 + xi_31 - _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]); + const double xi_157 = xi_156*0.125; + const double xi_159 = omega_shear*(xi_111 + xi_114*2.0 + xi_118 - xi_119 + xi_122 + xi_155 + xi_20 - 2.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 2.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_161 = xi_159*-0.0416666666666667 + xi_160*-0.166666666666667; + const double xi_162 = xi_104*-0.1 + xi_110*-0.05 + xi_161; + const double xi_163 = xi_107*0.0285714285714286 + xi_113*0.0142857142857143 + xi_154 + xi_157 + xi_158 + xi_162; + const double xi_177 = xi_107*-0.0714285714285714 + xi_113*-0.0357142857142857 + xi_158 + xi_159*0.0833333333333333 + xi_160*0.333333333333333; + const double xi_182 = rho*u_2 - vel2Term + xi_10 + xi_116 + xi_121 + xi_179 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_183 = xi_133*xi_182; + const double xi_191 = xi_103*0.0952380952380952 + xi_107*-0.0428571428571429 + xi_113*-0.0214285714285714 + xi_127*0.0158730158730159 - xi_154 - xi_157 + xi_162; + const double xi_194 = xi_156*0.0625; + const double xi_199 = xi_106*0.0833333333333333 + xi_123*0.0416666666666667; + const double xi_200 = xi_198 + xi_199; + const double xi_201 = xi_164 + xi_193 + xi_194 + xi_195 + xi_196 + xi_200; + const double xi_203 = xi_159*0.0208333333333333 + xi_160*0.0833333333333333; + const double xi_204 = -xi_202 + xi_203; + const double xi_205 = xi_178 + xi_204; + const double xi_211 = xi_202 + xi_203; + const double xi_212 = xi_176 + xi_211; + const double xi_213 = -xi_198 + xi_199; + const double xi_214 = xi_150 + xi_193 + xi_194 + xi_195 + xi_196 + xi_213; + const double xi_230 = xi_182*xi_197; + const double xi_232 = xi_230 + xi_231; + const double xi_233 = -xi_223 + xi_225 - xi_227 + xi_229 + xi_232; + const double xi_238 = xi_200 - xi_234 + xi_235 - xi_236 + xi_237; + const double xi_239 = xi_213 + xi_234 - xi_235 + xi_236 - xi_237; + const double xi_242 = -xi_194; + const double xi_245 = xi_192 + xi_199 + xi_232 + xi_241 + xi_242 + xi_243 + xi_244; + const double xi_253 = xi_204 - xi_249 + xi_250 - xi_251 + xi_252; + const double xi_255 = xi_211 + xi_249 - xi_250 + xi_251 - xi_252; + const double xi_256 = -xi_230 + xi_231; + const double xi_257 = xi_223 - xi_225 + xi_227 - xi_229 + xi_256; + const double xi_258 = xi_190 + xi_199 + xi_241 + xi_242 + xi_243 + xi_244 + xi_256; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_208 = xi_207*(-p_1 + u_0*xi_131); + const double xi_209 = -xi_206 - xi_208; + const double xi_210 = xi_206 + xi_208; + const double p_2 = xi_36; + const double xi_247 = xi_207*(-p_2 + u_2*xi_165); + const double xi_248 = -xi_246 - xi_247; + const double xi_254 = xi_246 + xi_247; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_217 = xi_207*(-p_5 + u_2*xi_131); + const double xi_221 = xi_161 + xi_215 + xi_216 + xi_217 + xi_218 + xi_219 + xi_220; + const double xi_240 = xi_161 - xi_215 + xi_216 - xi_217 + xi_218 + xi_219 + xi_220; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_3 + xi_7 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double forceTerm_0 = xi_47*-1.5 - xi_50*xi_53 - xi_53*xi_55 - xi_53*xi_56; + const double forceTerm_1 = xi_57 + xi_64; + const double forceTerm_2 = -xi_57 + xi_64; + const double forceTerm_3 = -xi_65 + xi_68; + const double forceTerm_4 = xi_65 + xi_68; + const double forceTerm_5 = xi_69 + xi_70; + const double forceTerm_6 = -xi_69 + xi_70; + const double forceTerm_7 = xi_75 + xi_77 + xi_84; + const double forceTerm_8 = xi_74 + xi_84 + xi_85; + const double forceTerm_9 = xi_74 + xi_77 + xi_87; + const double forceTerm_10 = xi_75 + xi_85 + xi_87; + const double forceTerm_11 = xi_83 + xi_88 + xi_92; + const double forceTerm_12 = xi_86 + xi_92 + xi_93; + const double forceTerm_13 = xi_77 + xi_95 + xi_97; + const double forceTerm_14 = xi_85 + xi_94 + xi_97; + const double forceTerm_15 = xi_83 + xi_93 + xi_99; + const double forceTerm_16 = xi_86 + xi_88 + xi_99; + const double forceTerm_17 = xi_100 + xi_77 + xi_94; + const double forceTerm_18 = xi_100 + xi_85 + xi_95; + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_0 + xi_103*0.142857142857143 + xi_104*0.2 - xi_106 + xi_107*0.0857142857142857 + xi_110*0.1 + xi_113*0.0428571428571429 + xi_123*-0.5 + xi_127*0.0238095238095238 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_1 - xi_130 + xi_134 - xi_139 + xi_150 + xi_163 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_2 + xi_130 - xi_134 + xi_139 + xi_163 + xi_164 + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_3 - xi_167 + xi_169 + xi_171 + xi_176 + xi_177 + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_4 + xi_167 - xi_169 - xi_171 + xi_177 + xi_178 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_5 - xi_181 + xi_183 - xi_185 + xi_190 + xi_191 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_6 + xi_181 - xi_183 + xi_185 + xi_191 + xi_192 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_7 + xi_201 + xi_205 + xi_209 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_8 + xi_201 + xi_210 + xi_212 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_9 + xi_205 + xi_210 + xi_214 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_10 + xi_209 + xi_212 + xi_214 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_11 + xi_221 + xi_233 + xi_238 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_12 + xi_233 + xi_239 + xi_240 + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_13 + xi_245 + xi_248 + xi_253 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_14 + xi_245 + xi_254 + xi_255 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_15 + xi_238 + xi_240 + xi_257 + _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_16 + xi_221 + xi_239 + xi_257 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_17 + xi_253 + xi_254 + xi_258 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_18 + xi_248 + xi_255 + xi_258 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_0, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, uint32_t block_offset_0, - uint32_t block_offset_1, uint32_t block_offset_2, double kT, - double omega_bulk, double omega_even, double omega_odd, - double omega_shear, uint32_t seed, uint32_t time_step) { - const double xi_25 = -omega_bulk; - const double xi_36 = -omega_shear; - const double xi_37 = xi_36 + 2.0; - const double xi_38 = xi_37 * 0.5; - const double xi_43 = xi_37 * 0.0833333333333333; - const double xi_48 = xi_37 * 0.166666666666667; - const double xi_58 = xi_37 * 0.25; - const double xi_63 = xi_37 * 0.0416666666666667; - const double xi_90 = 2.4494897427831779; - const double xi_115 = omega_odd * 0.25; - const double xi_131 = omega_odd * 0.0833333333333333; - const double xi_196 = omega_shear * 0.25; - const double xi_211 = omega_odd * 0.0416666666666667; - const double xi_213 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_120 = rr_0 * 0.166666666666667; - const double xi_186 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; - for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) { - const double xi_248 = _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0]; - const double xi_249 = _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0]; - const double xi_250 = _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0]; - const double xi_251 = _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0]; - const double xi_252 = _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_253 = _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0]; - const double xi_254 = _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0]; - const double xi_255 = _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_256 = _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0]; - const double xi_257 = _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0]; - const double xi_258 = _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0]; - const double xi_259 = _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_260 = _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_261 = _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_262 = _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0]; - const double xi_263 = _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0]; - const double xi_264 = _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0]; - const double xi_265 = _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0]; - const double xi_266 = _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0]; - const double xi_267 = _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0]; - const double xi_268 = _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_269 = _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0]; - - float random_3_0; - float random_3_1; - float random_3_2; - float random_3_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, - random_3_2, random_3_3); - - float random_2_0; - float random_2_1; - float random_2_2; - float random_2_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, - random_2_2, random_2_3); - - float random_1_0; - float random_1_1; - float random_1_2; - float random_1_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, - random_1_2, random_1_3); - - float random_0_0; - float random_0_1; - float random_0_2; - float random_0_3; - philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, - block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, - random_0_2, random_0_3); - - const double xi_0 = xi_256 + xi_262; - const double xi_1 = xi_0 + xi_253; - const double xi_2 = xi_250 + xi_264 + xi_265; - const double xi_3 = xi_248 + xi_259; - const double xi_4 = xi_257 + xi_258; - const double xi_5 = xi_249 + xi_269; - const double xi_6 = xi_260 + xi_266; - const double xi_8 = -xi_258; - const double xi_9 = -xi_254 + xi_8; - const double xi_10 = -xi_266; - const double xi_11 = -xi_251; - const double xi_12 = -xi_257; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -xi_269; - const double xi_15 = -xi_267; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -xi_249; - const double xi_18 = -xi_248; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = -xi_262; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -xi_265; - const double xi_23 = -xi_260; - const double xi_24 = xi_17 + xi_22 + xi_23 + xi_264; - const double xi_42 = xi_255 * 0.166666666666667; - const double xi_50 = xi_261 * 0.166666666666667; - const double xi_54 = xi_268 * 0.166666666666667; - const double xi_57 = xi_255 * 0.5; - const double xi_61 = xi_261 * 0.0833333333333333; - const double xi_65 = xi_255 * 0.0833333333333333; - const double xi_75 = xi_268 * 0.0833333333333333; - const double xi_93 = -xi_252; - const double xi_94 = xi_259 * 3.0 + xi_260 * 3.0 + xi_93; - const double xi_95 = - omega_even * (xi_248 * -3.0 + xi_249 * -3.0 + xi_250 * 3.0 + - xi_264 * -3.0 + xi_265 * -3.0 + xi_269 * 3.0 + xi_94); - const double xi_96 = - xi_248 * 2.0 + xi_249 * 2.0 + xi_264 * 2.0 + xi_265 * 2.0; - const double xi_97 = xi_253 * 5.0 + xi_257 * 5.0 + xi_96; - const double xi_98 = - omega_even * - (xi_250 * -2.0 + xi_251 * -5.0 + xi_256 * -5.0 + xi_262 * -5.0 + - xi_266 * -5.0 + xi_269 * -2.0 + xi_94 + xi_97); - const double xi_101 = -xi_264; - const double xi_102 = xi_101 + xi_18; - const double xi_103 = -xi_263; - const double xi_106 = -xi_256; - const double xi_107 = xi_106 + xi_11 + xi_15 + xi_21; - const double xi_109 = xi_251 * 2.0; - const double xi_110 = xi_256 * 2.0; - const double xi_111 = xi_262 * 2.0 + xi_266 * 2.0; - const double xi_112 = - omega_even * - (xi_109 + xi_110 + xi_111 + xi_250 * 5.0 + xi_254 * -7.0 + - xi_258 * -7.0 + xi_259 * -4.0 + xi_260 * -4.0 + xi_263 * -7.0 + - xi_267 * -7.0 + xi_269 * 5.0 + xi_93 + xi_97); - const double xi_113 = xi_101 + xi_248; - const double xi_114 = xi_113 + xi_14 + xi_22 + xi_249 + xi_250; - const double xi_116 = xi_114 * xi_115; - const double xi_118 = xi_103 + xi_267; - const double xi_122 = random_2_3 - 0.5; - const double xi_127 = xi_254 * 2.0; - const double xi_128 = xi_267 * 2.0; - const double xi_129 = xi_258 * 2.0 + xi_263 * -2.0; - const double xi_130 = -xi_127 + xi_128 + xi_129 + xi_14 + xi_19 + xi_2; - const double xi_132 = xi_130 * xi_131; - const double xi_133 = random_1_2 - 0.5; - const double xi_138 = random_0_1 - 0.5; - const double xi_142 = xi_251 + xi_266; - const double xi_156 = xi_106 + xi_251; - const double xi_157 = xi_12 + xi_156 + xi_20 + xi_253 + xi_266; - const double xi_158 = xi_115 * xi_157; - const double xi_159 = random_2_1 - 0.5; - const double xi_161 = xi_1 + xi_127 - xi_128 + xi_129 + xi_13; - const double xi_162 = xi_131 * xi_161; - const double xi_163 = random_2_0 - 0.5; - const double xi_168 = xi_249 + xi_265; - const double xi_169 = xi_102 + xi_168 + xi_23 + xi_259; - const double xi_170 = xi_115 * xi_169; - const double xi_173 = random_2_2 - 0.5; - const double xi_175 = -xi_109 - xi_110 + xi_111 + xi_24 + xi_3; - const double xi_176 = xi_131 * xi_175; - const double xi_177 = random_1_3 - 0.5; - const double xi_184 = xi_112 * 0.0138888888888889; - const double xi_205 = xi_98 * -0.00714285714285714; - const double xi_207 = xi_95 * 0.025; - const double xi_212 = xi_175 * xi_211; - const double xi_214 = xi_169 * xi_213; - const double xi_223 = xi_130 * xi_211; - const double xi_224 = xi_114 * xi_213; - const double xi_232 = xi_98 * 0.0178571428571429; - const double xi_238 = xi_157 * xi_213; - const double xi_239 = xi_161 * xi_211; - const double vel0Term = xi_1 + xi_263 + xi_267; - const double vel1Term = xi_2 + xi_254; - const double vel2Term = xi_251 + xi_3; - const double rho = - vel0Term + vel1Term + vel2Term + xi_252 + xi_4 + xi_5 + xi_6; - const double xi_7 = 1 / (rho); - const double xi_86 = kT * rho; - const double xi_87 = - sqrt(xi_86 * (-((-omega_even + 1.0) * (-omega_even + 1.0)) + 1.0)); - const double xi_88 = xi_87 * (random_3_0 - 0.5) * 3.7416573867739413; - const double xi_89 = xi_87 * (random_3_2 - 0.5) * 5.4772255750516612; - const double xi_91 = - xi_90 * sqrt(xi_86 * (-((xi_25 + 1.0) * (xi_25 + 1.0)) + 1.0)) * - (random_1_1 - 0.5); - const double xi_92 = xi_87 * (random_3_1 - 0.5) * 8.3666002653407556; - const double xi_123 = - sqrt(xi_86 * (-((-omega_odd + 1.0) * (-omega_odd + 1.0)) + 1.0)); - const double xi_124 = xi_123 * 1.4142135623730951; - const double xi_125 = xi_124 * 0.5; - const double xi_126 = xi_122 * xi_125; - const double xi_134 = xi_123 * xi_90; - const double xi_135 = xi_134 * 0.166666666666667; - const double xi_136 = xi_133 * xi_135; - const double xi_137 = -xi_132 - xi_136; - const double xi_139 = - sqrt(xi_86 * (-((xi_36 + 1.0) * (xi_36 + 1.0)) + 1.0)); - const double xi_140 = xi_139 * 0.5; - const double xi_141 = xi_138 * xi_140; - const double xi_146 = - xi_112 * -0.0198412698412698 + xi_88 * -0.119047619047619; - const double xi_148 = xi_139 * (random_0_0 - 0.5) * 1.7320508075688772; - const double xi_152 = xi_132 + xi_136; - const double xi_160 = xi_125 * xi_159; - const double xi_164 = xi_135 * xi_163; - const double xi_165 = xi_162 + xi_164; - const double xi_167 = -xi_162 - xi_164; - const double xi_174 = xi_125 * xi_173; - const double xi_178 = xi_135 * xi_177; - const double xi_179 = -xi_176 - xi_178; - const double xi_181 = xi_176 + xi_178; - const double xi_182 = xi_138 * xi_139 * 0.25; - const double xi_185 = xi_88 * 0.0833333333333333; - const double xi_195 = xi_140 * (random_0_2 - 0.5); - const double xi_204 = xi_140 * (random_1_0 - 0.5); - const double xi_208 = xi_92 * -0.0142857142857143; - const double xi_209 = xi_89 * 0.05; - const double xi_215 = xi_134 * 0.0833333333333333; - const double xi_216 = xi_177 * xi_215; - const double xi_217 = xi_124 * 0.25; - const double xi_218 = xi_173 * xi_217; - const double xi_220 = - xi_112 * -0.00396825396825397 + xi_88 * -0.0238095238095238; - const double xi_225 = xi_133 * xi_215; - const double xi_226 = xi_122 * xi_217; - const double xi_230 = -xi_182; - const double xi_233 = xi_92 * 0.0357142857142857; - const double xi_235 = xi_140 * (random_0_3 - 0.5); - const double xi_240 = xi_159 * xi_217; - const double xi_241 = xi_163 * xi_215; - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_26 = u_0 * xi_261; - const double xi_27 = xi_26 * 0.333333333333333; - const double xi_33 = -xi_27; - const double xi_99 = rho * (u_0 * u_0); - const double xi_153 = rho * u_0; - const double xi_154 = -vel0Term + xi_142 + xi_153 + xi_254 + xi_4; - const double xi_155 = xi_120 * xi_154; - const double xi_191 = xi_154 * xi_186; - const double u_1 = xi_7 * (vel1Term + xi_16 + xi_19 + xi_263 + xi_8); - const double xi_28 = u_1 * xi_255; - const double xi_29 = xi_28 * 0.333333333333333; - const double xi_34 = -xi_29; - const double xi_56 = u_1 * 0.5; - const double xi_59 = xi_58 * (u_0 * xi_57 + xi_261 * xi_56); - const double xi_60 = -xi_59; - const double xi_104 = rho * (u_1 * u_1); - const double xi_105 = xi_103 + xi_104 + xi_9; - const double xi_117 = rho * u_1; - const double xi_119 = - -vel1Term + xi_117 + xi_118 + xi_248 + xi_258 + xi_5; - const double xi_121 = xi_119 * xi_120; - const double xi_187 = xi_119 * xi_186; - const double xi_197 = xi_196 * (u_0 * xi_117 + xi_118 + xi_254 + xi_8); - const double xi_198 = -xi_195 - xi_197; - const double xi_199 = xi_195 + xi_197; - const double u_2 = xi_7 * (vel2Term + xi_21 + xi_24 + xi_256); - const double xi_30 = u_2 * xi_268; - const double xi_31 = xi_30 * 0.333333333333333; - const double xi_32 = (xi_25 + 2.0) * (xi_27 + xi_29 + xi_31); - const double xi_35 = xi_30 * 0.666666666666667 + xi_33 + xi_34; - const double xi_39 = -xi_31; - const double xi_40 = xi_28 * 0.666666666666667 + xi_33 + xi_39; - const double xi_41 = xi_26 * 0.666666666666667 + xi_34 + xi_39; - const double xi_44 = xi_35 * xi_43; - const double xi_45 = -xi_44; - const double xi_46 = xi_41 * xi_43; - const double xi_47 = -xi_46; - const double xi_49 = xi_40 * xi_48 + xi_45 + xi_47; - const double xi_51 = xi_40 * xi_43; - const double xi_52 = -xi_51; - const double xi_53 = xi_41 * xi_48 + xi_45 + xi_52; - const double xi_55 = xi_35 * xi_48 + xi_47 + xi_52; - const double xi_62 = xi_46 - xi_61; - const double xi_64 = -xi_35 * xi_63; - const double xi_66 = xi_32 * 0.125; - const double xi_67 = xi_51 + xi_66; - const double xi_68 = xi_65 + xi_67; - const double xi_69 = xi_64 + xi_68; - const double xi_70 = xi_46 + xi_61; - const double xi_71 = -xi_65 + xi_67; - const double xi_72 = xi_64 + xi_71; - const double xi_73 = xi_58 * (u_2 * xi_57 + xi_268 * xi_56); - const double xi_74 = -xi_41 * xi_63; - const double xi_76 = xi_44 + xi_75; - const double xi_77 = xi_74 + xi_76; - const double xi_78 = -xi_73; - const double xi_79 = xi_58 * (u_0 * xi_268 * 0.5 + u_2 * xi_261 * 0.5); - const double xi_80 = -xi_79; - const double xi_81 = -xi_40 * xi_63; - const double xi_82 = xi_66 + xi_76 + xi_81; - const double xi_83 = xi_44 - xi_75; - const double xi_84 = xi_74 + xi_83; - const double xi_85 = xi_66 + xi_81 + xi_83; - const double xi_100 = rho * (u_2 * u_2); - const double xi_108 = omega_bulk * (xi_100 + xi_102 + xi_105 + xi_107 + - xi_17 + xi_22 + xi_252 + xi_99); - const double xi_143 = -xi_100 + xi_259 + xi_260; - const double xi_144 = - omega_shear * (xi_0 + xi_105 + xi_142 + xi_143 + xi_16 - xi_250); - const double xi_145 = xi_144 * 0.125; - const double xi_147 = - omega_shear * - (xi_103 - xi_104 + xi_107 + xi_143 + xi_250 + xi_253 * -2.0 + - xi_257 * -2.0 + xi_269 + xi_9 + xi_96 + xi_99 * 2.0); - const double xi_149 = - xi_147 * -0.0416666666666667 + xi_148 * -0.166666666666667; - const double xi_150 = xi_149 + xi_89 * -0.1 + xi_95 * -0.05; - const double xi_151 = xi_141 + xi_145 + xi_146 + xi_150 + - xi_92 * 0.0285714285714286 + - xi_98 * 0.0142857142857143; - const double xi_166 = - xi_146 + xi_147 * 0.0833333333333333 + xi_148 * 0.333333333333333 + - xi_92 * -0.0714285714285714 + xi_98 * -0.0357142857142857; - const double xi_171 = - rho * u_2 - vel2Term + xi_101 + xi_106 + xi_168 + xi_262 + xi_6; - const double xi_172 = xi_120 * xi_171; - const double xi_180 = xi_112 * 0.0158730158730159 - xi_141 - xi_145 + - xi_150 + xi_88 * 0.0952380952380952 + - xi_92 * -0.0428571428571429 + - xi_98 * -0.0214285714285714; - const double xi_183 = xi_144 * 0.0625; - const double xi_188 = - xi_108 * 0.0416666666666667 + xi_91 * 0.0833333333333333; - const double xi_189 = xi_187 + xi_188; - const double xi_190 = - xi_152 + xi_182 + xi_183 + xi_184 + xi_185 + xi_189; - const double xi_192 = - xi_147 * 0.0208333333333333 + xi_148 * 0.0833333333333333; - const double xi_193 = -xi_191 + xi_192; - const double xi_194 = xi_167 + xi_193; - const double xi_200 = xi_191 + xi_192; - const double xi_201 = xi_165 + xi_200; - const double xi_202 = -xi_187 + xi_188; - const double xi_203 = - xi_137 + xi_182 + xi_183 + xi_184 + xi_185 + xi_202; - const double xi_206 = xi_196 * (u_2 * xi_117 + xi_113 + xi_17 + xi_265); - const double xi_210 = - xi_149 + xi_204 + xi_205 + xi_206 + xi_207 + xi_208 + xi_209; - const double xi_219 = xi_171 * xi_186; - const double xi_221 = xi_219 + xi_220; - const double xi_222 = -xi_212 + xi_214 - xi_216 + xi_218 + xi_221; - const double xi_227 = xi_189 - xi_223 + xi_224 - xi_225 + xi_226; - const double xi_228 = xi_202 + xi_223 - xi_224 + xi_225 - xi_226; - const double xi_229 = - xi_149 - xi_204 + xi_205 - xi_206 + xi_207 + xi_208 + xi_209; - const double xi_231 = -xi_183; - const double xi_234 = - xi_181 + xi_188 + xi_221 + xi_230 + xi_231 + xi_232 + xi_233; - const double xi_236 = xi_196 * (u_2 * xi_153 + xi_10 + xi_156 + xi_262); - const double xi_237 = -xi_235 - xi_236; - const double xi_242 = xi_193 - xi_238 + xi_239 - xi_240 + xi_241; - const double xi_243 = xi_235 + xi_236; - const double xi_244 = xi_200 + xi_238 - xi_239 + xi_240 - xi_241; - const double xi_245 = -xi_219 + xi_220; - const double xi_246 = xi_212 - xi_214 + xi_216 - xi_218 + xi_245; - const double xi_247 = - xi_179 + xi_188 + xi_230 + xi_231 + xi_232 + xi_233 + xi_245; - const double forceTerm_0 = - xi_32 * -1.5 - xi_35 * xi_38 - xi_38 * xi_40 - xi_38 * xi_41; - const double forceTerm_1 = xi_42 + xi_49; - const double forceTerm_2 = -xi_42 + xi_49; - const double forceTerm_3 = -xi_50 + xi_53; - const double forceTerm_4 = xi_50 + xi_53; - const double forceTerm_5 = xi_54 + xi_55; - const double forceTerm_6 = -xi_54 + xi_55; - const double forceTerm_7 = xi_60 + xi_62 + xi_69; - const double forceTerm_8 = xi_59 + xi_69 + xi_70; - const double forceTerm_9 = xi_59 + xi_62 + xi_72; - const double forceTerm_10 = xi_60 + xi_70 + xi_72; - const double forceTerm_11 = xi_68 + xi_73 + xi_77; - const double forceTerm_12 = xi_71 + xi_77 + xi_78; - const double forceTerm_13 = xi_62 + xi_80 + xi_82; - const double forceTerm_14 = xi_70 + xi_79 + xi_82; - const double forceTerm_15 = xi_68 + xi_78 + xi_84; - const double forceTerm_16 = xi_71 + xi_73 + xi_84; - const double forceTerm_17 = xi_62 + xi_79 + xi_85; - const double forceTerm_18 = xi_70 + xi_80 + xi_85; - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0] = - forceTerm_0 + xi_108 * -0.5 + xi_112 * 0.0238095238095238 + xi_252 + - xi_88 * 0.142857142857143 + xi_89 * 0.2 - xi_91 + - xi_92 * 0.0857142857142857 + xi_95 * 0.1 + - xi_98 * 0.0428571428571429; - _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0] = - forceTerm_1 - xi_116 + xi_121 - xi_126 + xi_137 + xi_151 + xi_250; - _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0] = - forceTerm_2 + xi_116 - xi_121 + xi_126 + xi_151 + xi_152 + xi_269; - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0] = - forceTerm_3 - xi_155 + xi_158 + xi_160 + xi_165 + xi_166 + xi_257; - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0] = - forceTerm_4 + xi_155 - xi_158 - xi_160 + xi_166 + xi_167 + xi_253; - _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0] = - forceTerm_5 - xi_170 + xi_172 - xi_174 + xi_179 + xi_180 + xi_259; - _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0] = - forceTerm_6 + xi_170 - xi_172 + xi_174 + xi_180 + xi_181 + xi_260; - _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0] = - forceTerm_7 + xi_190 + xi_194 + xi_198 + xi_254; - _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0] = - forceTerm_8 + xi_190 + xi_199 + xi_201 + xi_263; - _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0] = - forceTerm_9 + xi_194 + xi_199 + xi_203 + xi_258; - _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0] = - forceTerm_10 + xi_198 + xi_201 + xi_203 + xi_267; - _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0] = - forceTerm_11 + xi_210 + xi_222 + xi_227 + xi_264; - _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0] = - forceTerm_12 + xi_222 + xi_228 + xi_229 + xi_248; - _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0] = - forceTerm_13 + xi_234 + xi_237 + xi_242 + xi_251; - _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0] = - forceTerm_14 + xi_234 + xi_243 + xi_244 + xi_256; - _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0] = - forceTerm_15 + xi_227 + xi_229 + xi_246 + xi_265; - _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0] = - forceTerm_16 + xi_210 + xi_228 + xi_246 + xi_249; - _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0] = - forceTerm_17 + xi_242 + xi_243 + xi_247 + xi_266; - _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0] = - forceTerm_18 + xi_237 + xi_244 + xi_247 + xi_262; +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, uint32_t block_offset_0, uint32_t block_offset_1, uint32_t block_offset_2, double kT, double omega_bulk, double omega_even, double omega_odd, double omega_shear, uint32_t seed, uint32_t time_step) +{ + const double xi_40 = -omega_bulk; + const double xi_51 = -omega_shear; + const double xi_52 = xi_51 + 2.0; + const double xi_53 = xi_52*0.5; + const double xi_58 = xi_52*0.0833333333333333; + const double xi_63 = xi_52*0.166666666666667; + const double xi_73 = xi_52*0.25; + const double xi_78 = xi_52*0.0416666666666667; + const double xi_105 = 2.4494897427831779; + const double xi_129 = omega_odd*0.25; + const double xi_144 = omega_odd*0.0833333333333333; + const double xi_207 = omega_shear*0.25; + const double xi_222 = omega_odd*0.0416666666666667; + const double xi_224 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_133 = rr_0*0.166666666666667; + const double xi_197 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) + { + const double xi_259 = _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0]; + const double xi_260 = _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0]; + const double xi_261 = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0]; + const double xi_262 = _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0]; + const double xi_263 = _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0]; + const double xi_264 = _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0]; + const double xi_265 = _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0]; + const double xi_266 = _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0]; + const double xi_267 = _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0]; + const double xi_268 = _data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_269 = _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0]; + const double xi_270 = _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0]; + const double xi_271 = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_272 = _data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_273 = _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0]; + const double xi_274 = _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0]; + const double xi_275 = _data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_276 = _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0]; + const double xi_277 = _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0]; + const double xi_278 = _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0]; + const double xi_279 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0]; + const double xi_280 = _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0]; + + float random_3_0; + float random_3_1; + float random_3_2; + float random_3_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 3, seed, random_3_0, random_3_1, random_3_2, random_3_3); + + + float random_2_0; + float random_2_1; + float random_2_2; + float random_2_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 2, seed, random_2_0, random_2_1, random_2_2, random_2_3); + + + float random_1_0; + float random_1_1; + float random_1_2; + float random_1_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 1, seed, random_1_0, random_1_1, random_1_2, random_1_3); + + + float random_0_0; + float random_0_1; + float random_0_2; + float random_0_3; + philox_float4(time_step, block_offset_0 + ctr_0, block_offset_1 + ctr_1, block_offset_2 + ctr_2, 0, seed, random_0_0, random_0_1, random_0_2, random_0_3); + + const double xi_0 = xi_263 + xi_273; + const double xi_1 = xi_260 + xi_266; + const double xi_2 = xi_0 + xi_1 + xi_261; + const double xi_3 = xi_265 + xi_270; + const double xi_4 = xi_276 + xi_3; + const double xi_5 = xi_280 + xi_4; + const double xi_6 = xi_274 + xi_277; + const double xi_7 = xi_267 + xi_6; + const double xi_8 = xi_269 + xi_279; + const double xi_9 = xi_259 + xi_262; + const double xi_10 = xi_264 + xi_278; + const double xi_12 = -xi_278; + const double xi_13 = -xi_279; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -xi_267; + const double xi_16 = -xi_280; + const double xi_17 = -xi_269; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -xi_273; + const double xi_21 = xi_20 + xi_263; + const double xi_22 = -xi_277; + const double xi_23 = -xi_259; + const double xi_24 = -xi_262; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -xi_266; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -xi_270; + const double xi_29 = -xi_264; + const double xi_30 = xi_24 + xi_265 + xi_28 + xi_29; + const double xi_31 = xi_267 + xi_278; + const double xi_32 = xi_280 + xi_31 + xi_8; + const double xi_33 = xi_16 + xi_21 + xi_269; + const double xi_34 = xi_26 + xi_278; + const double xi_35 = xi_15 + xi_260; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_269 + xi_277 + xi_9; + const double xi_38 = xi_262 + xi_28; + const double xi_39 = xi_22 + xi_265 + xi_38; + const double xi_57 = xi_272*0.166666666666667; + const double xi_65 = xi_268*0.166666666666667; + const double xi_69 = xi_275*0.166666666666667; + const double xi_72 = xi_272*0.5; + const double xi_76 = xi_268*0.0833333333333333; + const double xi_80 = xi_272*0.0833333333333333; + const double xi_90 = xi_275*0.0833333333333333; + const double xi_108 = -xi_271; + const double xi_109 = xi_108 + xi_264*3.0 + xi_274*3.0; + const double xi_110 = omega_even*(xi_109 + xi_259*3.0 + xi_262*-3.0 + xi_265*-3.0 + xi_270*-3.0 + xi_276*3.0 + xi_277*-3.0); + const double xi_111 = xi_262*2.0 + xi_265*2.0 + xi_270*2.0 + xi_277*2.0; + const double xi_112 = xi_111 + xi_261*5.0 + xi_279*5.0; + const double xi_113 = omega_even*(xi_109 + xi_112 + xi_259*-2.0 + xi_260*-5.0 + xi_266*-5.0 + xi_267*-5.0 + xi_276*-2.0 + xi_278*-5.0); + const double xi_116 = -xi_265; + const double xi_117 = xi_116 + xi_22; + const double xi_118 = -xi_263; + const double xi_121 = -xi_260; + const double xi_122 = xi_121 + xi_19 + xi_27; + const double xi_124 = xi_267*2.0; + const double xi_125 = xi_260*2.0; + const double xi_126 = xi_266*2.0 + xi_278*2.0; + const double xi_127 = omega_even*(xi_108 + xi_112 + xi_124 + xi_125 + xi_126 + xi_259*5.0 + xi_263*-7.0 + xi_264*-4.0 + xi_269*-7.0 + xi_273*-7.0 + xi_274*-4.0 + xi_276*5.0 + xi_280*-7.0); + const double xi_128 = xi_116 + xi_23 + xi_276 + xi_277 + xi_38; + const double xi_130 = xi_128*xi_129; + const double xi_135 = random_2_3 - 0.5; + const double xi_140 = xi_280*2.0; + const double xi_141 = xi_273*2.0; + const double xi_142 = xi_263*-2.0 + xi_269*2.0; + const double xi_143 = -xi_140 + xi_141 + xi_142 + xi_25 + xi_4; + const double xi_145 = xi_143*xi_144; + const double xi_146 = random_1_2 - 0.5; + const double xi_151 = random_0_1 - 0.5; + const double xi_168 = xi_121 + xi_13 + xi_261 + xi_267 + xi_34; + const double xi_169 = xi_129*xi_168; + const double xi_170 = random_2_1 - 0.5; + const double xi_172 = xi_14 + xi_140 - xi_141 + xi_142 + xi_261 + xi_266 + xi_35; + const double xi_173 = xi_144*xi_172; + const double xi_174 = random_2_0 - 0.5; + const double xi_179 = xi_262 + xi_270; + const double xi_180 = xi_117 + xi_179 + xi_274 + xi_29; + const double xi_181 = xi_129*xi_180; + const double xi_184 = random_2_2 - 0.5; + const double xi_186 = -xi_124 - xi_125 + xi_126 + xi_30 + xi_6; + const double xi_187 = xi_144*xi_186; + const double xi_188 = random_1_3 - 0.5; + const double xi_195 = xi_127*0.0138888888888889; + const double xi_216 = xi_113*-0.00714285714285714; + const double xi_218 = xi_110*0.025; + const double xi_223 = xi_186*xi_222; + const double xi_225 = xi_180*xi_224; + const double xi_234 = xi_143*xi_222; + const double xi_235 = xi_128*xi_224; + const double xi_243 = xi_113*0.0178571428571429; + const double xi_249 = xi_168*xi_224; + const double xi_250 = xi_172*xi_222; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_271 + xi_8 + xi_9; + const double xi_11 = 1 / (rho); + const double xi_101 = kT*rho; + const double xi_102 = sqrt(xi_101*(-((-omega_even + 1.0)*(-omega_even + 1.0)) + 1.0)); + const double xi_103 = xi_102*(random_3_0 - 0.5)*3.7416573867739413; + const double xi_104 = xi_102*(random_3_2 - 0.5)*5.4772255750516612; + const double xi_106 = xi_105*sqrt(xi_101*(-((xi_40 + 1.0)*(xi_40 + 1.0)) + 1.0))*(random_1_1 - 0.5); + const double xi_107 = xi_102*(random_3_1 - 0.5)*8.3666002653407556; + const double xi_136 = sqrt(xi_101*(-((-omega_odd + 1.0)*(-omega_odd + 1.0)) + 1.0)); + const double xi_137 = xi_136*1.4142135623730951; + const double xi_138 = xi_137*0.5; + const double xi_139 = xi_135*xi_138; + const double xi_147 = xi_105*xi_136; + const double xi_148 = xi_147*0.166666666666667; + const double xi_149 = xi_146*xi_148; + const double xi_150 = -xi_145 - xi_149; + const double xi_152 = sqrt(xi_101*(-((xi_51 + 1.0)*(xi_51 + 1.0)) + 1.0)); + const double xi_153 = xi_152*0.5; + const double xi_154 = xi_151*xi_153; + const double xi_158 = xi_103*-0.119047619047619 + xi_127*-0.0198412698412698; + const double xi_160 = xi_152*(random_0_0 - 0.5)*1.7320508075688772; + const double xi_164 = xi_145 + xi_149; + const double xi_171 = xi_138*xi_170; + const double xi_175 = xi_148*xi_174; + const double xi_176 = xi_173 + xi_175; + const double xi_178 = -xi_173 - xi_175; + const double xi_185 = xi_138*xi_184; + const double xi_189 = xi_148*xi_188; + const double xi_190 = -xi_187 - xi_189; + const double xi_192 = xi_187 + xi_189; + const double xi_193 = xi_151*xi_152*0.25; + const double xi_196 = xi_103*0.0833333333333333; + const double xi_206 = xi_153*(random_0_2 - 0.5); + const double xi_215 = xi_153*(random_1_0 - 0.5); + const double xi_219 = xi_107*-0.0142857142857143; + const double xi_220 = xi_104*0.05; + const double xi_226 = xi_147*0.0833333333333333; + const double xi_227 = xi_188*xi_226; + const double xi_228 = xi_137*0.25; + const double xi_229 = xi_184*xi_228; + const double xi_231 = xi_103*-0.0238095238095238 + xi_127*-0.00396825396825397; + const double xi_236 = xi_146*xi_226; + const double xi_237 = xi_135*xi_228; + const double xi_241 = -xi_193; + const double xi_244 = xi_107*0.0357142857142857; + const double xi_246 = xi_153*(random_0_3 - 0.5); + const double xi_251 = xi_170*xi_228; + const double xi_252 = xi_174*xi_226; + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_41 = u_0*xi_268; + const double xi_42 = xi_41*0.333333333333333; + const double xi_48 = -xi_42; + const double xi_114 = rho*(u_0*u_0); + const double xi_165 = rho*u_0; + const double xi_166 = -vel0Term + xi_165 + xi_32; + const double xi_167 = xi_133*xi_166; + const double xi_202 = xi_166*xi_197; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_43 = u_1*xi_272; + const double xi_44 = xi_43*0.333333333333333; + const double xi_49 = -xi_44; + const double xi_71 = u_1*0.5; + const double xi_74 = xi_73*(u_0*xi_72 + xi_268*xi_71); + const double xi_75 = -xi_74; + const double xi_119 = rho*(u_1*u_1); + const double xi_120 = xi_118 + xi_119 + xi_20; + const double xi_131 = rho*u_1; + const double xi_132 = -vel1Term + xi_118 + xi_131 + xi_273 + xi_37; + const double xi_134 = xi_132*xi_133; + const double xi_198 = xi_132*xi_197; + const double u_2 = xi_11*(vel2Term + xi_260 + xi_27 + xi_30); + const double xi_45 = u_2*xi_275; + const double xi_46 = xi_45*0.333333333333333; + const double xi_47 = (xi_40 + 2.0)*(xi_42 + xi_44 + xi_46); + const double xi_50 = xi_45*0.666666666666667 + xi_48 + xi_49; + const double xi_54 = -xi_46; + const double xi_55 = xi_43*0.666666666666667 + xi_48 + xi_54; + const double xi_56 = xi_41*0.666666666666667 + xi_49 + xi_54; + const double xi_59 = xi_50*xi_58; + const double xi_60 = -xi_59; + const double xi_61 = xi_56*xi_58; + const double xi_62 = -xi_61; + const double xi_64 = xi_55*xi_63 + xi_60 + xi_62; + const double xi_66 = xi_55*xi_58; + const double xi_67 = -xi_66; + const double xi_68 = xi_56*xi_63 + xi_60 + xi_67; + const double xi_70 = xi_50*xi_63 + xi_62 + xi_67; + const double xi_77 = xi_61 - xi_76; + const double xi_79 = -xi_50*xi_78; + const double xi_81 = xi_47*0.125; + const double xi_82 = xi_66 + xi_81; + const double xi_83 = xi_80 + xi_82; + const double xi_84 = xi_79 + xi_83; + const double xi_85 = xi_61 + xi_76; + const double xi_86 = -xi_80 + xi_82; + const double xi_87 = xi_79 + xi_86; + const double xi_88 = xi_73*(u_2*xi_72 + xi_275*xi_71); + const double xi_89 = -xi_56*xi_78; + const double xi_91 = xi_59 + xi_90; + const double xi_92 = xi_89 + xi_91; + const double xi_93 = -xi_88; + const double xi_94 = xi_73*(u_0*xi_275*0.5 + u_2*xi_268*0.5); + const double xi_95 = -xi_94; + const double xi_96 = -xi_55*xi_78; + const double xi_97 = xi_81 + xi_91 + xi_96; + const double xi_98 = xi_59 - xi_90; + const double xi_99 = xi_89 + xi_98; + const double xi_100 = xi_81 + xi_96 + xi_98; + const double xi_115 = rho*(u_2*u_2); + const double xi_123 = omega_bulk*(xi_114 + xi_115 + xi_117 + xi_120 + xi_122 + xi_24 + xi_271 + xi_28); + const double xi_155 = -xi_115 + xi_264 + xi_274; + const double xi_156 = omega_shear*(xi_1 + xi_120 + xi_155 + xi_18 + xi_23 - xi_276 + xi_31); + const double xi_157 = xi_156*0.125; + const double xi_159 = omega_shear*(xi_111 + xi_114*2.0 + xi_118 - xi_119 + xi_122 + xi_155 + xi_20 + xi_259 + xi_261*-2.0 + xi_276 + xi_279*-2.0); + const double xi_161 = xi_159*-0.0416666666666667 + xi_160*-0.166666666666667; + const double xi_162 = xi_104*-0.1 + xi_110*-0.05 + xi_161; + const double xi_163 = xi_107*0.0285714285714286 + xi_113*0.0142857142857143 + xi_154 + xi_157 + xi_158 + xi_162; + const double xi_177 = xi_107*-0.0714285714285714 + xi_113*-0.0357142857142857 + xi_158 + xi_159*0.0833333333333333 + xi_160*0.333333333333333; + const double xi_182 = rho*u_2 - vel2Term + xi_10 + xi_116 + xi_121 + xi_179 + xi_266; + const double xi_183 = xi_133*xi_182; + const double xi_191 = xi_103*0.0952380952380952 + xi_107*-0.0428571428571429 + xi_113*-0.0214285714285714 + xi_127*0.0158730158730159 - xi_154 - xi_157 + xi_162; + const double xi_194 = xi_156*0.0625; + const double xi_199 = xi_106*0.0833333333333333 + xi_123*0.0416666666666667; + const double xi_200 = xi_198 + xi_199; + const double xi_201 = xi_164 + xi_193 + xi_194 + xi_195 + xi_196 + xi_200; + const double xi_203 = xi_159*0.0208333333333333 + xi_160*0.0833333333333333; + const double xi_204 = -xi_202 + xi_203; + const double xi_205 = xi_178 + xi_204; + const double xi_211 = xi_202 + xi_203; + const double xi_212 = xi_176 + xi_211; + const double xi_213 = -xi_198 + xi_199; + const double xi_214 = xi_150 + xi_193 + xi_194 + xi_195 + xi_196 + xi_213; + const double xi_230 = xi_182*xi_197; + const double xi_232 = xi_230 + xi_231; + const double xi_233 = -xi_223 + xi_225 - xi_227 + xi_229 + xi_232; + const double xi_238 = xi_200 - xi_234 + xi_235 - xi_236 + xi_237; + const double xi_239 = xi_213 + xi_234 - xi_235 + xi_236 - xi_237; + const double xi_242 = -xi_194; + const double xi_245 = xi_192 + xi_199 + xi_232 + xi_241 + xi_242 + xi_243 + xi_244; + const double xi_253 = xi_204 - xi_249 + xi_250 - xi_251 + xi_252; + const double xi_255 = xi_211 + xi_249 - xi_250 + xi_251 - xi_252; + const double xi_256 = -xi_230 + xi_231; + const double xi_257 = xi_223 - xi_225 + xi_227 - xi_229 + xi_256; + const double xi_258 = xi_190 + xi_199 + xi_241 + xi_242 + xi_243 + xi_244 + xi_256; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_208 = xi_207*(-p_1 + u_0*xi_131); + const double xi_209 = -xi_206 - xi_208; + const double xi_210 = xi_206 + xi_208; + const double p_2 = xi_36; + const double xi_247 = xi_207*(-p_2 + u_2*xi_165); + const double xi_248 = -xi_246 - xi_247; + const double xi_254 = xi_246 + xi_247; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_217 = xi_207*(-p_5 + u_2*xi_131); + const double xi_221 = xi_161 + xi_215 + xi_216 + xi_217 + xi_218 + xi_219 + xi_220; + const double xi_240 = xi_161 - xi_215 + xi_216 - xi_217 + xi_218 + xi_219 + xi_220; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_262 + xi_3 + xi_7; + const double forceTerm_0 = xi_47*-1.5 - xi_50*xi_53 - xi_53*xi_55 - xi_53*xi_56; + const double forceTerm_1 = xi_57 + xi_64; + const double forceTerm_2 = -xi_57 + xi_64; + const double forceTerm_3 = -xi_65 + xi_68; + const double forceTerm_4 = xi_65 + xi_68; + const double forceTerm_5 = xi_69 + xi_70; + const double forceTerm_6 = -xi_69 + xi_70; + const double forceTerm_7 = xi_75 + xi_77 + xi_84; + const double forceTerm_8 = xi_74 + xi_84 + xi_85; + const double forceTerm_9 = xi_74 + xi_77 + xi_87; + const double forceTerm_10 = xi_75 + xi_85 + xi_87; + const double forceTerm_11 = xi_83 + xi_88 + xi_92; + const double forceTerm_12 = xi_86 + xi_92 + xi_93; + const double forceTerm_13 = xi_77 + xi_95 + xi_97; + const double forceTerm_14 = xi_85 + xi_94 + xi_97; + const double forceTerm_15 = xi_83 + xi_93 + xi_99; + const double forceTerm_16 = xi_86 + xi_88 + xi_99; + const double forceTerm_17 = xi_100 + xi_77 + xi_94; + const double forceTerm_18 = xi_100 + xi_85 + xi_95; + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0] = forceTerm_0 + xi_103*0.142857142857143 + xi_104*0.2 - xi_106 + xi_107*0.0857142857142857 + xi_110*0.1 + xi_113*0.0428571428571429 + xi_123*-0.5 + xi_127*0.0238095238095238 + xi_271; + _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0] = forceTerm_1 - xi_130 + xi_134 - xi_139 + xi_150 + xi_163 + xi_276; + _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0] = forceTerm_2 + xi_130 - xi_134 + xi_139 + xi_163 + xi_164 + xi_259; + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0] = forceTerm_3 - xi_167 + xi_169 + xi_171 + xi_176 + xi_177 + xi_279; + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0] = forceTerm_4 + xi_167 - xi_169 - xi_171 + xi_177 + xi_178 + xi_261; + _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0] = forceTerm_5 - xi_181 + xi_183 - xi_185 + xi_190 + xi_191 + xi_274; + _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0] = forceTerm_6 + xi_181 - xi_183 + xi_185 + xi_191 + xi_192 + xi_264; + _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0] = forceTerm_7 + xi_201 + xi_205 + xi_209 + xi_280; + _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0] = forceTerm_8 + xi_201 + xi_210 + xi_212 + xi_263; + _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0] = forceTerm_9 + xi_205 + xi_210 + xi_214 + xi_269; + _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0] = forceTerm_10 + xi_209 + xi_212 + xi_214 + xi_273; + _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0] = forceTerm_11 + xi_221 + xi_233 + xi_238 + xi_265; + _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0] = forceTerm_12 + xi_233 + xi_239 + xi_240 + xi_277; + _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0] = forceTerm_13 + xi_245 + xi_248 + xi_253 + xi_267; + _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0] = forceTerm_14 + xi_245 + xi_254 + xi_255 + xi_260; + _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0] = forceTerm_15 + xi_238 + xi_240 + xi_257 + xi_270; + _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0] = forceTerm_16 + xi_221 + xi_239 + xi_257 + xi_262; + _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0] = forceTerm_17 + xi_253 + xi_254 + xi_258 + xi_278; + _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0] = forceTerm_18 + xi_248 + xi_255 + xi_258 + xi_266; + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void kernel_stream( - double *RESTRICT const _data_pdfs, double *RESTRICT _data_pdfs_tmp, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) { - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) + { + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } } -} // namespace internal_kernel_stream +} + -const real_t FluctuatingMRTLatticeModelAvx::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t FluctuatingMRTLatticeModelAvx::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; +const real_t FluctuatingMRTLatticeModelAvx::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t FluctuatingMRTLatticeModelAvx::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; -void FluctuatingMRTLatticeModelAvx::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +void FluctuatingMRTLatticeModelAvx::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &seed = lm.seed_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - auto &block_offset_2 = lm.block_offset_2_; - auto &block_offset_0 = lm.block_offset_0_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, - _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, - _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, - block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, - time_step); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_even = lm.omega_even_; + auto & block_offset_1 = lm.block_offset_1_; + auto & seed = lm.seed_; + auto & time_step = lm.time_step_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & block_offset_2 = lm.block_offset_2_; + auto & omega_odd = lm.omega_odd_; + auto & kT = lm.kT_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); + pdfs->swapDataPointers(pdfs_tmp); + } -void FluctuatingMRTLatticeModelAvx::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void FluctuatingMRTLatticeModelAvx::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); - auto &lm = dynamic_cast *>(pdfs) - ->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &seed = lm.seed_; - auto &time_step = lm.time_step_; - auto &block_offset_1 = lm.block_offset_1_; - auto &omega_shear = lm.omega_shear_; - auto &kT = lm.kT_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - auto &block_offset_2 = lm.block_offset_2_; - auto &block_offset_0 = lm.block_offset_0_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, - omega_even, omega_odd, omega_shear, seed, time_step); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & block_offset_0 = lm.block_offset_0_; + auto & omega_even = lm.omega_even_; + auto & block_offset_1 = lm.block_offset_1_; + auto & seed = lm.seed_; + auto & time_step = lm.time_step_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & block_offset_2 = lm.block_offset_2_; + auto & omega_odd = lm.omega_odd_; + auto & kT = lm.kT_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, block_offset_0, block_offset_1, block_offset_2, kT, omega_bulk, omega_even, omega_odd, omega_shear, seed, time_step); } -void FluctuatingMRTLatticeModelAvx::Sweep::stream( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void FluctuatingMRTLatticeModelAvx::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, - _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer & -operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::FluctuatingMRTLatticeModelAvx &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::FluctuatingMRTLatticeModelAvx & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer & -operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::FluctuatingMRTLatticeModelAvx &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::FluctuatingMRTLatticeModelAvx & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h index 139c83920b0..04fd6fc90b4 100644 --- a/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h +++ b/src/walberla_bridge/generated_kernels/FluctuatingMRTLatticeModelAvx.h @@ -696,16 +696,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModelAvx & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const FluctuatingMRTLatticeModelAvx & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const FluctuatingMRTLatticeModelAvx & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const FluctuatingMRTLatticeModelAvx & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp b/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp index 44bad1e18e6..f2bcca056ca 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModel.cpp @@ -18,1521 +18,1011 @@ #include -#include "MRTLatticeModel.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "MRTLatticeModel.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_0, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, - double omega_bulk, double omega_even, double omega_odd, - double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) { - const double xi_0 = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_1 = - xi_0 + _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_2 = _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_3 = _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_4 = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_5 = _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_6 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_8 = - -_data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_9 = - xi_8 - - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_10 = - -_data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_11 = - -_data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_12 = - -_data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -_data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - const double xi_15 = - -_data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -_data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_18 = -_data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = - -_data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -_data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_23 = -_data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_24 = xi_17 + xi_22 + xi_23 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_40 = - 0.166666666666667 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_48 = - 0.166666666666667 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_52 = - 0.166666666666667 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_55 = - 0.5 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_59 = - 0.0833333333333333 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_63 = - 0.0833333333333333 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_73 = - 0.0833333333333333 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_84 = -_data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_85 = xi_84 + - 3.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_86 = - omega_even * - (xi_85 - 3.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] - - 3.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 3.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_87 = - 2.0 * _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0] + - 2.0 * _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_88 = - xi_87 + - 5.0 * _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_89 = - omega_even * - (xi_85 + xi_88 - - 2.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] - - 2.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0] - - 5.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 5.0 * - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + - _stride_pdfs_0] - - 5.0 * _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - - _stride_pdfs_0]); - const double xi_92 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - const double xi_93 = xi_18 + xi_92; - const double xi_94 = - -_data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_97 = - -_data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_98 = xi_11 + xi_15 + xi_21 + xi_97; - const double xi_100 = - 2.0 * - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_101 = - 2.0 * - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_102 = - 2.0 * - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 2.0 * _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_103 = - omega_even * - (xi_100 + xi_101 + xi_102 + xi_84 + xi_88 - - 4.0 * _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] - - 4.0 * _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0] - - 7.0 * - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] - - 7.0 * - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - 5.0 * _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - 5.0 * _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_104 = - xi_92 + _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_105 = xi_104 + xi_14 + xi_22 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_107 = xi_105 * xi_106; - const double xi_108 = - 2.0 * _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_109 = - 2.0 * _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_110 = - -2.0 * - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - 2.0 * _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_111 = -xi_108 + xi_109 + xi_110 + xi_14 + xi_19 + xi_2; - const double xi_113 = xi_111 * xi_112; - const double xi_114 = -xi_113; - const double xi_116 = - xi_94 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_120 = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_124 = xi_103 * -0.0198412698412698; - const double xi_132 = - xi_97 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_133 = - xi_12 + xi_132 + xi_20 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_134 = xi_106 * xi_133; - const double xi_135 = xi_1 + xi_108 - xi_109 + xi_110 + xi_13; - const double xi_136 = xi_112 * xi_135; - const double xi_138 = -xi_136; - const double xi_139 = _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - const double xi_140 = xi_139 + xi_23 + xi_93 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_141 = xi_106 * xi_140; - const double xi_144 = -xi_100 - xi_101 + xi_102 + xi_24 + xi_3; - const double xi_145 = xi_112 * xi_144; - const double xi_146 = -xi_145; - const double xi_148 = xi_145; - const double xi_152 = xi_103 * 0.0138888888888889; - const double xi_168 = xi_89 * -0.00714285714285714; - const double xi_170 = xi_86 * 0.025; - const double xi_173 = xi_144 * xi_172; - const double xi_175 = xi_140 * xi_174; - const double xi_176 = xi_103 * -0.00396825396825397; - const double xi_180 = xi_111 * xi_172; - const double xi_181 = xi_105 * xi_174; - const double xi_187 = xi_89 * 0.0178571428571429; - const double xi_190 = xi_133 * xi_174; - const double xi_191 = xi_135 * xi_172; - const double vel0Term = - xi_1 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double vel1Term = - xi_2 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double vel2Term = - xi_3 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double rho = vel0Term + vel1Term + vel2Term + xi_4 + xi_5 + xi_6 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_7 = 1 / (rho); - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_25 = - u_0 * _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_26 = xi_25 * 0.333333333333333; - const double xi_32 = -xi_26; - const double xi_90 = rho * (u_0 * u_0); - const double xi_129 = rho * u_0; - const double xi_130 = - -vel0Term + xi_120 + xi_129 + xi_4 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - const double xi_131 = xi_118 * xi_130; - const double xi_158 = xi_130 * xi_154; - const double u_1 = - xi_7 * - (vel1Term + xi_16 + xi_19 + xi_8 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_27 = - u_1 * _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_28 = xi_27 * 0.333333333333333; - const double xi_33 = -xi_28; - const double xi_54 = u_1 * 0.5; - const double xi_57 = - xi_56 * (u_0 * xi_55 + - xi_54 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_58 = -xi_57; - const double xi_95 = rho * (u_1 * u_1); - const double xi_96 = xi_9 + xi_94 + xi_95; - const double xi_115 = rho * u_1; - const double xi_117 = - -vel1Term + xi_115 + xi_116 + xi_5 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - const double xi_119 = xi_117 * xi_118; - const double xi_150 = - xi_149 * - (u_0 * xi_115 + xi_116 + xi_8 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]); - const double xi_155 = xi_117 * xi_154; - const double xi_156 = xi_155; - const double xi_157 = xi_113 + xi_156; - const double xi_166 = -xi_155; - const double xi_167 = xi_114 + xi_166; - const double xi_182 = xi_156 - xi_180 + xi_181; - const double xi_183 = xi_166 + xi_180 - xi_181; - const double u_2 = - xi_7 * - (vel2Term + xi_21 + xi_24 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_29 = - u_2 * _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_30 = xi_29 * 0.333333333333333; - const double xi_31 = (-omega_bulk + 2.0) * (xi_26 + xi_28 + xi_30); - const double xi_34 = xi_29 * 0.666666666666667 + xi_32 + xi_33; - const double xi_37 = -xi_30; - const double xi_38 = xi_27 * 0.666666666666667 + xi_32 + xi_37; - const double xi_39 = xi_25 * 0.666666666666667 + xi_33 + xi_37; - const double xi_42 = xi_34 * xi_41; - const double xi_43 = -xi_42; - const double xi_44 = xi_39 * xi_41; - const double xi_45 = -xi_44; - const double xi_47 = xi_38 * xi_46 + xi_43 + xi_45; - const double xi_49 = xi_38 * xi_41; - const double xi_50 = -xi_49; - const double xi_51 = xi_39 * xi_46 + xi_43 + xi_50; - const double xi_53 = xi_34 * xi_46 + xi_45 + xi_50; - const double xi_60 = xi_44 - xi_59; - const double xi_62 = -xi_34 * xi_61; - const double xi_64 = xi_31 * 0.125; - const double xi_65 = xi_49 + xi_64; - const double xi_66 = xi_63 + xi_65; - const double xi_67 = xi_62 + xi_66; - const double xi_68 = xi_44 + xi_59; - const double xi_69 = -xi_63 + xi_65; - const double xi_70 = xi_62 + xi_69; - const double xi_71 = - xi_56 * (u_2 * xi_55 + - xi_54 * _data_force_20_32_10[_stride_force_0 * ctr_0]); - const double xi_72 = -xi_39 * xi_61; - const double xi_74 = xi_42 + xi_73; - const double xi_75 = xi_72 + xi_74; - const double xi_76 = -xi_71; - const double xi_77 = - xi_56 * (u_0 * 0.5 * _data_force_20_32_10[_stride_force_0 * ctr_0] + - u_2 * 0.5 * _data_force_20_30_10[_stride_force_0 * ctr_0]); - const double xi_78 = -xi_77; - const double xi_79 = -xi_38 * xi_61; - const double xi_80 = xi_64 + xi_74 + xi_79; - const double xi_81 = xi_42 - xi_73; - const double xi_82 = xi_72 + xi_81; - const double xi_83 = xi_64 + xi_79 + xi_81; - const double xi_91 = rho * (u_2 * u_2); - const double xi_99 = - omega_bulk * (xi_17 + xi_22 + xi_90 + xi_91 + xi_93 + xi_96 + - xi_98 + _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]); - const double xi_121 = -xi_91 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0] + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_122 = - omega_shear * (xi_0 + xi_120 + xi_121 + xi_16 + xi_96 - - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_123 = xi_122 * 0.125; - const double xi_125 = - omega_shear * - (xi_121 + xi_87 + xi_9 + xi_90 * 2.0 + xi_94 - xi_95 + xi_98 - - 2.0 * - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] - - 2.0 * - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0] + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]); - const double xi_126 = xi_125 * -0.0416666666666667; - const double xi_127 = xi_126 + xi_86 * -0.05; - const double xi_128 = - xi_123 + xi_124 + xi_127 + xi_89 * 0.0142857142857143; - const double xi_137 = - xi_124 + xi_125 * 0.0833333333333333 + xi_89 * -0.0357142857142857; - const double xi_142 = - rho * u_2 - vel2Term + xi_139 + xi_6 + xi_92 + xi_97 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - const double xi_143 = xi_118 * xi_142; - const double xi_147 = xi_103 * 0.0158730158730159 - xi_123 + xi_127 + - xi_89 * -0.0214285714285714; - const double xi_151 = xi_122 * 0.0625; - const double xi_153 = -xi_150 + xi_151 + xi_152; - const double xi_159 = xi_99 * 0.0416666666666667; - const double xi_160 = xi_125 * 0.0208333333333333 + xi_159; - const double xi_161 = -xi_158 + xi_160; - const double xi_162 = xi_138 + xi_161; - const double xi_163 = xi_150 + xi_151 + xi_152; - const double xi_164 = xi_158 + xi_160; - const double xi_165 = xi_136 + xi_164; - const double xi_169 = - xi_149 * (u_2 * xi_115 + xi_104 + xi_17 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]); - const double xi_171 = xi_126 + xi_159 + xi_168 + xi_169 + xi_170; - const double xi_177 = xi_142 * xi_154; - const double xi_178 = xi_176 + xi_177; - const double xi_179 = -xi_173 + xi_175 + xi_178; - const double xi_184 = xi_126 + xi_159 + xi_168 - xi_169 + xi_170; - const double xi_185 = - xi_149 * - (u_2 * xi_129 + xi_10 + xi_132 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]); - const double xi_186 = -xi_151; - const double xi_188 = -xi_185 + xi_186 + xi_187; - const double xi_189 = xi_148 + xi_178; - const double xi_192 = xi_161 - xi_190 + xi_191; - const double xi_193 = xi_185 + xi_186 + xi_187; - const double xi_194 = xi_164 + xi_190 - xi_191; - const double xi_195 = xi_176 - xi_177; - const double xi_196 = xi_173 - xi_175 + xi_195; - const double xi_197 = xi_146 + xi_195; - const double forceTerm_0 = - xi_31 * -1.5 - xi_34 * xi_36 - xi_36 * xi_38 - xi_36 * xi_39; - const double forceTerm_1 = xi_40 + xi_47; - const double forceTerm_2 = -xi_40 + xi_47; - const double forceTerm_3 = -xi_48 + xi_51; - const double forceTerm_4 = xi_48 + xi_51; - const double forceTerm_5 = xi_52 + xi_53; - const double forceTerm_6 = -xi_52 + xi_53; - const double forceTerm_7 = xi_58 + xi_60 + xi_67; - const double forceTerm_8 = xi_57 + xi_67 + xi_68; - const double forceTerm_9 = xi_57 + xi_60 + xi_70; - const double forceTerm_10 = xi_58 + xi_68 + xi_70; - const double forceTerm_11 = xi_66 + xi_71 + xi_75; - const double forceTerm_12 = xi_69 + xi_75 + xi_76; - const double forceTerm_13 = xi_60 + xi_78 + xi_80; - const double forceTerm_14 = xi_68 + xi_77 + xi_80; - const double forceTerm_15 = xi_66 + xi_76 + xi_82; - const double forceTerm_16 = xi_69 + xi_71 + xi_82; - const double forceTerm_17 = xi_60 + xi_77 + xi_83; - const double forceTerm_18 = xi_68 + xi_78 + xi_83; - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_0 + xi_103 * 0.0238095238095238 + xi_86 * 0.1 + - xi_89 * 0.0428571428571429 + xi_99 * -0.5 + - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_1 - xi_107 + xi_114 + xi_119 + xi_128 + - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_2 + xi_107 + xi_113 - xi_119 + xi_128 + - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_3 - xi_131 + xi_134 + xi_136 + xi_137 + - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_4 + xi_131 - xi_134 + xi_137 + xi_138 + - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_5 - xi_141 + xi_143 + xi_146 + xi_147 + - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_6 + xi_141 - xi_143 + xi_147 + xi_148 + - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_7 + xi_153 + xi_157 + xi_162 + - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_8 + xi_157 + xi_163 + xi_165 + - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_9 + xi_162 + xi_163 + xi_167 + - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_10 + xi_153 + xi_165 + xi_167 + - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_11 + xi_171 + xi_179 + xi_182 + - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_12 + xi_179 + xi_183 + xi_184 + - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_13 + xi_188 + xi_189 + xi_192 + - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_14 + xi_189 + xi_193 + xi_194 + - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_15 + xi_182 + xi_184 + xi_196 + - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_16 + xi_171 + xi_183 + xi_196 + - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_17 + xi_192 + xi_193 + xi_197 + - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - forceTerm_18 + xi_188 + xi_194 + xi_197 + - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) + { + const double xi_0 = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_1 = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_2 = xi_0 + xi_1 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_3 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_4 = xi_3 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + const double xi_5 = xi_4 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_6 = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_7 = xi_6 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_8 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_9 = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_10 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_12 = -_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_13 = -_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_16 = -_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_17 = -_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_21 = xi_20 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_22 = -_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_23 = -_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + const double xi_24 = -_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + const double xi_29 = -_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + const double xi_30 = xi_24 + xi_28 + xi_29 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_31 = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_32 = xi_31 + xi_8 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_33 = xi_16 + xi_21 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_34 = xi_26 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_35 = xi_15 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_9 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_38 = xi_28 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_39 = xi_22 + xi_38 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_55 = 0.166666666666667*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_63 = 0.166666666666667*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_67 = 0.166666666666667*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_70 = 0.5*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_74 = 0.0833333333333333*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_78 = 0.0833333333333333*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_88 = 0.0833333333333333*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_99 = -_data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_100 = xi_99 + 3.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_101 = omega_even*(xi_100 - 3.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] - 3.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 3.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_102 = 2.0*_data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_21_316_11[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0] + 2.0*_data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_103 = xi_102 + 5.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_104 = omega_even*(xi_100 + xi_103 - 2.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] - 2.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0] - 5.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 5.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 5.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_107 = -_data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + const double xi_108 = xi_107 + xi_22; + const double xi_109 = -_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_112 = -_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_113 = xi_112 + xi_19 + xi_27; + const double xi_115 = 2.0*_data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_116 = 2.0*_data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_117 = 2.0*_data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 2.0*_data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_118 = omega_even*(xi_103 + xi_115 + xi_116 + xi_117 + xi_99 - 4.0*_data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] - 4.0*_data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0] - 7.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 7.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] - 7.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] + 5.0*_data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + 5.0*_data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_119 = xi_107 + xi_23 + xi_38 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + const double xi_121 = xi_119*xi_120; + const double xi_122 = 2.0*_data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_123 = 2.0*_data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_124 = -2.0*_data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + 2.0*_data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_125 = -xi_122 + xi_123 + xi_124 + xi_25 + xi_4; + const double xi_127 = xi_125*xi_126; + const double xi_128 = -xi_127; + const double xi_133 = xi_118*-0.0198412698412698; + const double xi_144 = xi_112 + xi_13 + xi_34 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + const double xi_145 = xi_120*xi_144; + const double xi_146 = xi_122 - xi_123 + xi_124 + xi_14 + xi_35 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_147 = xi_126*xi_146; + const double xi_149 = -xi_147; + const double xi_150 = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double xi_151 = xi_108 + xi_150 + xi_29 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_152 = xi_120*xi_151; + const double xi_153 = -xi_115 - xi_116 + xi_117 + xi_30 + xi_6; + const double xi_154 = xi_126*xi_153; + const double xi_155 = -xi_154; + const double xi_162 = xi_118*0.0138888888888889; + const double xi_177 = xi_104*-0.00714285714285714; + const double xi_179 = xi_101*0.025; + const double xi_182 = xi_153*xi_181; + const double xi_184 = xi_151*xi_183; + const double xi_185 = xi_118*-0.00396825396825397; + const double xi_189 = xi_125*xi_181; + const double xi_190 = xi_119*xi_183; + const double xi_196 = xi_104*0.0178571428571429; + const double xi_199 = xi_144*xi_183; + const double xi_200 = xi_146*xi_181; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_8 + xi_9 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_11 = 1 / (rho); + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_40 = u_0*_data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_41 = xi_40*0.333333333333333; + const double xi_47 = -xi_41; + const double xi_105 = rho*(u_0*u_0); + const double xi_141 = rho*u_0; + const double xi_142 = -vel0Term + xi_141 + xi_32; + const double xi_143 = xi_131*xi_142; + const double xi_167 = xi_142*xi_164; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_42 = u_1*_data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_43 = xi_42*0.333333333333333; + const double xi_48 = -xi_43; + const double xi_69 = u_1*0.5; + const double xi_72 = xi_71*(u_0*xi_70 + xi_69*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_73 = -xi_72; + const double xi_110 = rho*(u_1*u_1); + const double xi_111 = xi_109 + xi_110 + xi_20; + const double xi_129 = rho*u_1; + const double xi_130 = -vel1Term + xi_109 + xi_129 + xi_37 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_132 = xi_130*xi_131; + const double xi_165 = xi_130*xi_164; + const double xi_166 = xi_127 + xi_165; + const double xi_175 = -xi_165; + const double xi_176 = xi_128 + xi_175; + const double xi_191 = xi_165 - xi_189 + xi_190; + const double xi_192 = xi_175 + xi_189 - xi_190; + const double u_2 = xi_11*(vel2Term + xi_27 + xi_30 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]); + const double xi_44 = u_2*_data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_45 = xi_44*0.333333333333333; + const double xi_46 = (-omega_bulk + 2.0)*(xi_41 + xi_43 + xi_45); + const double xi_49 = xi_44*0.666666666666667 + xi_47 + xi_48; + const double xi_52 = -xi_45; + const double xi_53 = xi_42*0.666666666666667 + xi_47 + xi_52; + const double xi_54 = xi_40*0.666666666666667 + xi_48 + xi_52; + const double xi_57 = xi_49*xi_56; + const double xi_58 = -xi_57; + const double xi_59 = xi_54*xi_56; + const double xi_60 = -xi_59; + const double xi_62 = xi_53*xi_61 + xi_58 + xi_60; + const double xi_64 = xi_53*xi_56; + const double xi_65 = -xi_64; + const double xi_66 = xi_54*xi_61 + xi_58 + xi_65; + const double xi_68 = xi_49*xi_61 + xi_60 + xi_65; + const double xi_75 = xi_59 - xi_74; + const double xi_77 = -xi_49*xi_76; + const double xi_79 = xi_46*0.125; + const double xi_80 = xi_64 + xi_79; + const double xi_81 = xi_78 + xi_80; + const double xi_82 = xi_77 + xi_81; + const double xi_83 = xi_59 + xi_74; + const double xi_84 = -xi_78 + xi_80; + const double xi_85 = xi_77 + xi_84; + const double xi_86 = xi_71*(u_2*xi_70 + xi_69*_data_force_20_32_10[_stride_force_0*ctr_0]); + const double xi_87 = -xi_54*xi_76; + const double xi_89 = xi_57 + xi_88; + const double xi_90 = xi_87 + xi_89; + const double xi_91 = -xi_86; + const double xi_92 = xi_71*(u_0*0.5*_data_force_20_32_10[_stride_force_0*ctr_0] + u_2*0.5*_data_force_20_30_10[_stride_force_0*ctr_0]); + const double xi_93 = -xi_92; + const double xi_94 = -xi_53*xi_76; + const double xi_95 = xi_79 + xi_89 + xi_94; + const double xi_96 = xi_57 - xi_88; + const double xi_97 = xi_87 + xi_96; + const double xi_98 = xi_79 + xi_94 + xi_96; + const double xi_106 = rho*(u_2*u_2); + const double xi_114 = omega_bulk*(xi_105 + xi_106 + xi_108 + xi_111 + xi_113 + xi_24 + xi_28 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]); + const double xi_134 = -xi_106 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0] + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + const double xi_135 = omega_shear*(xi_1 + xi_111 + xi_134 + xi_18 + xi_23 + xi_31 - _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]); + const double xi_136 = xi_135*0.125; + const double xi_137 = omega_shear*(xi_102 + xi_105*2.0 + xi_109 - xi_110 + xi_113 + xi_134 + xi_20 - 2.0*_data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] - 2.0*_data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0] + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]); + const double xi_138 = xi_137*-0.0416666666666667; + const double xi_139 = xi_101*-0.05 + xi_138; + const double xi_140 = xi_104*0.0142857142857143 + xi_133 + xi_136 + xi_139; + const double xi_148 = xi_104*-0.0357142857142857 + xi_133 + xi_137*0.0833333333333333; + const double xi_156 = rho*u_2 - vel2Term + xi_10 + xi_107 + xi_112 + xi_150 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + const double xi_157 = xi_131*xi_156; + const double xi_158 = xi_104*-0.0214285714285714 + xi_118*0.0158730158730159 - xi_136 + xi_139; + const double xi_161 = xi_135*0.0625; + const double xi_168 = xi_114*0.0416666666666667; + const double xi_169 = xi_137*0.0208333333333333 + xi_168; + const double xi_170 = -xi_167 + xi_169; + const double xi_171 = xi_149 + xi_170; + const double xi_173 = xi_167 + xi_169; + const double xi_174 = xi_147 + xi_173; + const double xi_186 = xi_156*xi_164; + const double xi_187 = xi_185 + xi_186; + const double xi_188 = -xi_182 + xi_184 + xi_187; + const double xi_195 = -xi_161; + const double xi_198 = xi_154 + xi_187; + const double xi_201 = xi_170 - xi_199 + xi_200; + const double xi_203 = xi_173 + xi_199 - xi_200; + const double xi_204 = xi_185 - xi_186; + const double xi_205 = xi_182 - xi_184 + xi_204; + const double xi_206 = xi_155 + xi_204; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_160 = xi_159*(-p_1 + u_0*xi_129); + const double xi_163 = -xi_160 + xi_161 + xi_162; + const double xi_172 = xi_160 + xi_161 + xi_162; + const double p_2 = xi_36; + const double xi_194 = xi_159*(-p_2 + u_2*xi_141); + const double xi_197 = -xi_194 + xi_195 + xi_196; + const double xi_202 = xi_194 + xi_195 + xi_196; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_178 = xi_159*(-p_5 + u_2*xi_129); + const double xi_180 = xi_138 + xi_168 + xi_177 + xi_178 + xi_179; + const double xi_193 = xi_138 + xi_168 + xi_177 - xi_178 + xi_179; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_3 + xi_7 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + const double forceTerm_0 = xi_46*-1.5 - xi_49*xi_51 - xi_51*xi_53 - xi_51*xi_54; + const double forceTerm_1 = xi_55 + xi_62; + const double forceTerm_2 = -xi_55 + xi_62; + const double forceTerm_3 = -xi_63 + xi_66; + const double forceTerm_4 = xi_63 + xi_66; + const double forceTerm_5 = xi_67 + xi_68; + const double forceTerm_6 = -xi_67 + xi_68; + const double forceTerm_7 = xi_73 + xi_75 + xi_82; + const double forceTerm_8 = xi_72 + xi_82 + xi_83; + const double forceTerm_9 = xi_72 + xi_75 + xi_85; + const double forceTerm_10 = xi_73 + xi_83 + xi_85; + const double forceTerm_11 = xi_81 + xi_86 + xi_90; + const double forceTerm_12 = xi_84 + xi_90 + xi_91; + const double forceTerm_13 = xi_75 + xi_93 + xi_95; + const double forceTerm_14 = xi_83 + xi_92 + xi_95; + const double forceTerm_15 = xi_81 + xi_91 + xi_97; + const double forceTerm_16 = xi_84 + xi_86 + xi_97; + const double forceTerm_17 = xi_75 + xi_92 + xi_98; + const double forceTerm_18 = xi_83 + xi_93 + xi_98; + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_0 + xi_101*0.1 + xi_104*0.0428571428571429 + xi_114*-0.5 + xi_118*0.0238095238095238 + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_1 - xi_121 + xi_128 + xi_132 + xi_140 + _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_2 + xi_121 + xi_127 - xi_132 + xi_140 + _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_3 - xi_143 + xi_145 + xi_147 + xi_148 + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_4 + xi_143 - xi_145 + xi_148 + xi_149 + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_5 - xi_152 + xi_155 + xi_157 + xi_158 + _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_6 + xi_152 + xi_154 - xi_157 + xi_158 + _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_7 + xi_163 + xi_166 + xi_171 + _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_8 + xi_166 + xi_172 + xi_174 + _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_9 + xi_171 + xi_172 + xi_176 + _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_10 + xi_163 + xi_174 + xi_176 + _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_11 + xi_180 + xi_188 + xi_191 + _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_12 + xi_188 + xi_192 + xi_193 + _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_13 + xi_197 + xi_198 + xi_201 + _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_14 + xi_198 + xi_202 + xi_203 + _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_15 + xi_191 + xi_193 + xi_205 + _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_16 + xi_180 + xi_192 + xi_205 + _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_17 + xi_201 + xi_202 + xi_206 + _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = forceTerm_18 + xi_197 + xi_203 + xi_206 + _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_0, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, double omega_bulk, - double omega_even, double omega_odd, double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; - for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) { - const double xi_198 = _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0]; - const double xi_199 = _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0]; - const double xi_200 = _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0]; - const double xi_201 = _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0]; - const double xi_202 = _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - const double xi_203 = _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0]; - const double xi_204 = _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0]; - const double xi_205 = _data_force_20_31_10[_stride_force_0 * ctr_0]; - const double xi_206 = _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0]; - const double xi_207 = _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0]; - const double xi_208 = _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0]; - const double xi_209 = _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0]; - const double xi_210 = _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0]; - const double xi_211 = _data_force_20_30_10[_stride_force_0 * ctr_0]; - const double xi_212 = _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0]; - const double xi_213 = _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0]; - const double xi_214 = _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0]; - const double xi_215 = _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0]; - const double xi_216 = _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0]; - const double xi_217 = _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0]; - const double xi_218 = _data_force_20_32_10[_stride_force_0 * ctr_0]; - const double xi_219 = _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0]; - const double xi_0 = xi_206 + xi_212; - const double xi_1 = xi_0 + xi_203; - const double xi_2 = xi_200 + xi_214 + xi_215; - const double xi_3 = xi_198 + xi_209; - const double xi_4 = xi_207 + xi_208; - const double xi_5 = xi_199 + xi_219; - const double xi_6 = xi_210 + xi_216; - const double xi_8 = -xi_208; - const double xi_9 = -xi_204 + xi_8; - const double xi_10 = -xi_216; - const double xi_11 = -xi_201; - const double xi_12 = -xi_207; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -xi_219; - const double xi_15 = -xi_217; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -xi_199; - const double xi_18 = -xi_198; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = -xi_212; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -xi_215; - const double xi_23 = -xi_210; - const double xi_24 = xi_17 + xi_214 + xi_22 + xi_23; - const double xi_40 = xi_205 * 0.166666666666667; - const double xi_48 = xi_211 * 0.166666666666667; - const double xi_52 = xi_218 * 0.166666666666667; - const double xi_55 = xi_205 * 0.5; - const double xi_59 = xi_211 * 0.0833333333333333; - const double xi_63 = xi_205 * 0.0833333333333333; - const double xi_73 = xi_218 * 0.0833333333333333; - const double xi_84 = -xi_202; - const double xi_85 = xi_209 * 3.0 + xi_210 * 3.0 + xi_84; - const double xi_86 = - omega_even * (xi_198 * -3.0 + xi_199 * -3.0 + xi_200 * 3.0 + - xi_214 * -3.0 + xi_215 * -3.0 + xi_219 * 3.0 + xi_85); - const double xi_87 = - xi_198 * 2.0 + xi_199 * 2.0 + xi_214 * 2.0 + xi_215 * 2.0; - const double xi_88 = xi_203 * 5.0 + xi_207 * 5.0 + xi_87; - const double xi_89 = - omega_even * - (xi_200 * -2.0 + xi_201 * -5.0 + xi_206 * -5.0 + xi_212 * -5.0 + - xi_216 * -5.0 + xi_219 * -2.0 + xi_85 + xi_88); - const double xi_92 = -xi_214; - const double xi_93 = xi_18 + xi_92; - const double xi_94 = -xi_213; - const double xi_97 = -xi_206; - const double xi_98 = xi_11 + xi_15 + xi_21 + xi_97; - const double xi_100 = xi_201 * 2.0; - const double xi_101 = xi_206 * 2.0; - const double xi_102 = xi_212 * 2.0 + xi_216 * 2.0; - const double xi_103 = - omega_even * - (xi_100 + xi_101 + xi_102 + xi_200 * 5.0 + xi_204 * -7.0 + - xi_208 * -7.0 + xi_209 * -4.0 + xi_210 * -4.0 + xi_213 * -7.0 + - xi_217 * -7.0 + xi_219 * 5.0 + xi_84 + xi_88); - const double xi_104 = xi_198 + xi_92; - const double xi_105 = xi_104 + xi_14 + xi_199 + xi_200 + xi_22; - const double xi_107 = xi_105 * xi_106; - const double xi_108 = xi_204 * 2.0; - const double xi_109 = xi_217 * 2.0; - const double xi_110 = xi_208 * 2.0 + xi_213 * -2.0; - const double xi_111 = -xi_108 + xi_109 + xi_110 + xi_14 + xi_19 + xi_2; - const double xi_113 = xi_111 * xi_112; - const double xi_114 = -xi_113; - const double xi_116 = xi_217 + xi_94; - const double xi_120 = xi_201 + xi_216; - const double xi_124 = xi_103 * -0.0198412698412698; - const double xi_132 = xi_201 + xi_97; - const double xi_133 = xi_12 + xi_132 + xi_20 + xi_203 + xi_216; - const double xi_134 = xi_106 * xi_133; - const double xi_135 = xi_1 + xi_108 - xi_109 + xi_110 + xi_13; - const double xi_136 = xi_112 * xi_135; - const double xi_138 = -xi_136; - const double xi_139 = xi_199 + xi_215; - const double xi_140 = xi_139 + xi_209 + xi_23 + xi_93; - const double xi_141 = xi_106 * xi_140; - const double xi_144 = -xi_100 - xi_101 + xi_102 + xi_24 + xi_3; - const double xi_145 = xi_112 * xi_144; - const double xi_146 = -xi_145; - const double xi_148 = xi_145; - const double xi_152 = xi_103 * 0.0138888888888889; - const double xi_168 = xi_89 * -0.00714285714285714; - const double xi_170 = xi_86 * 0.025; - const double xi_173 = xi_144 * xi_172; - const double xi_175 = xi_140 * xi_174; - const double xi_176 = xi_103 * -0.00396825396825397; - const double xi_180 = xi_111 * xi_172; - const double xi_181 = xi_105 * xi_174; - const double xi_187 = xi_89 * 0.0178571428571429; - const double xi_190 = xi_133 * xi_174; - const double xi_191 = xi_135 * xi_172; - const double vel0Term = xi_1 + xi_213 + xi_217; - const double vel1Term = xi_2 + xi_204; - const double vel2Term = xi_201 + xi_3; - const double rho = - vel0Term + vel1Term + vel2Term + xi_202 + xi_4 + xi_5 + xi_6; - const double xi_7 = 1 / (rho); - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_25 = u_0 * xi_211; - const double xi_26 = xi_25 * 0.333333333333333; - const double xi_32 = -xi_26; - const double xi_90 = rho * (u_0 * u_0); - const double xi_129 = rho * u_0; - const double xi_130 = -vel0Term + xi_120 + xi_129 + xi_204 + xi_4; - const double xi_131 = xi_118 * xi_130; - const double xi_158 = xi_130 * xi_154; - const double u_1 = xi_7 * (vel1Term + xi_16 + xi_19 + xi_213 + xi_8); - const double xi_27 = u_1 * xi_205; - const double xi_28 = xi_27 * 0.333333333333333; - const double xi_33 = -xi_28; - const double xi_54 = u_1 * 0.5; - const double xi_57 = xi_56 * (u_0 * xi_55 + xi_211 * xi_54); - const double xi_58 = -xi_57; - const double xi_95 = rho * (u_1 * u_1); - const double xi_96 = xi_9 + xi_94 + xi_95; - const double xi_115 = rho * u_1; - const double xi_117 = - -vel1Term + xi_115 + xi_116 + xi_198 + xi_208 + xi_5; - const double xi_119 = xi_117 * xi_118; - const double xi_150 = xi_149 * (u_0 * xi_115 + xi_116 + xi_204 + xi_8); - const double xi_155 = xi_117 * xi_154; - const double xi_156 = xi_155; - const double xi_157 = xi_113 + xi_156; - const double xi_166 = -xi_155; - const double xi_167 = xi_114 + xi_166; - const double xi_182 = xi_156 - xi_180 + xi_181; - const double xi_183 = xi_166 + xi_180 - xi_181; - const double u_2 = xi_7 * (vel2Term + xi_206 + xi_21 + xi_24); - const double xi_29 = u_2 * xi_218; - const double xi_30 = xi_29 * 0.333333333333333; - const double xi_31 = (-omega_bulk + 2.0) * (xi_26 + xi_28 + xi_30); - const double xi_34 = xi_29 * 0.666666666666667 + xi_32 + xi_33; - const double xi_37 = -xi_30; - const double xi_38 = xi_27 * 0.666666666666667 + xi_32 + xi_37; - const double xi_39 = xi_25 * 0.666666666666667 + xi_33 + xi_37; - const double xi_42 = xi_34 * xi_41; - const double xi_43 = -xi_42; - const double xi_44 = xi_39 * xi_41; - const double xi_45 = -xi_44; - const double xi_47 = xi_38 * xi_46 + xi_43 + xi_45; - const double xi_49 = xi_38 * xi_41; - const double xi_50 = -xi_49; - const double xi_51 = xi_39 * xi_46 + xi_43 + xi_50; - const double xi_53 = xi_34 * xi_46 + xi_45 + xi_50; - const double xi_60 = xi_44 - xi_59; - const double xi_62 = -xi_34 * xi_61; - const double xi_64 = xi_31 * 0.125; - const double xi_65 = xi_49 + xi_64; - const double xi_66 = xi_63 + xi_65; - const double xi_67 = xi_62 + xi_66; - const double xi_68 = xi_44 + xi_59; - const double xi_69 = -xi_63 + xi_65; - const double xi_70 = xi_62 + xi_69; - const double xi_71 = xi_56 * (u_2 * xi_55 + xi_218 * xi_54); - const double xi_72 = -xi_39 * xi_61; - const double xi_74 = xi_42 + xi_73; - const double xi_75 = xi_72 + xi_74; - const double xi_76 = -xi_71; - const double xi_77 = xi_56 * (u_0 * xi_218 * 0.5 + u_2 * xi_211 * 0.5); - const double xi_78 = -xi_77; - const double xi_79 = -xi_38 * xi_61; - const double xi_80 = xi_64 + xi_74 + xi_79; - const double xi_81 = xi_42 - xi_73; - const double xi_82 = xi_72 + xi_81; - const double xi_83 = xi_64 + xi_79 + xi_81; - const double xi_91 = rho * (u_2 * u_2); - const double xi_99 = omega_bulk * (xi_17 + xi_202 + xi_22 + xi_90 + - xi_91 + xi_93 + xi_96 + xi_98); - const double xi_121 = xi_209 + xi_210 - xi_91; - const double xi_122 = - omega_shear * (xi_0 + xi_120 + xi_121 + xi_16 - xi_200 + xi_96); - const double xi_123 = xi_122 * 0.125; - const double xi_125 = - omega_shear * - (xi_121 + xi_200 + xi_203 * -2.0 + xi_207 * -2.0 + xi_219 + xi_87 + - xi_9 + xi_90 * 2.0 + xi_94 - xi_95 + xi_98); - const double xi_126 = xi_125 * -0.0416666666666667; - const double xi_127 = xi_126 + xi_86 * -0.05; - const double xi_128 = - xi_123 + xi_124 + xi_127 + xi_89 * 0.0142857142857143; - const double xi_137 = - xi_124 + xi_125 * 0.0833333333333333 + xi_89 * -0.0357142857142857; - const double xi_142 = - rho * u_2 - vel2Term + xi_139 + xi_212 + xi_6 + xi_92 + xi_97; - const double xi_143 = xi_118 * xi_142; - const double xi_147 = xi_103 * 0.0158730158730159 - xi_123 + xi_127 + - xi_89 * -0.0214285714285714; - const double xi_151 = xi_122 * 0.0625; - const double xi_153 = -xi_150 + xi_151 + xi_152; - const double xi_159 = xi_99 * 0.0416666666666667; - const double xi_160 = xi_125 * 0.0208333333333333 + xi_159; - const double xi_161 = -xi_158 + xi_160; - const double xi_162 = xi_138 + xi_161; - const double xi_163 = xi_150 + xi_151 + xi_152; - const double xi_164 = xi_158 + xi_160; - const double xi_165 = xi_136 + xi_164; - const double xi_169 = xi_149 * (u_2 * xi_115 + xi_104 + xi_17 + xi_215); - const double xi_171 = xi_126 + xi_159 + xi_168 + xi_169 + xi_170; - const double xi_177 = xi_142 * xi_154; - const double xi_178 = xi_176 + xi_177; - const double xi_179 = -xi_173 + xi_175 + xi_178; - const double xi_184 = xi_126 + xi_159 + xi_168 - xi_169 + xi_170; - const double xi_185 = xi_149 * (u_2 * xi_129 + xi_10 + xi_132 + xi_212); - const double xi_186 = -xi_151; - const double xi_188 = -xi_185 + xi_186 + xi_187; - const double xi_189 = xi_148 + xi_178; - const double xi_192 = xi_161 - xi_190 + xi_191; - const double xi_193 = xi_185 + xi_186 + xi_187; - const double xi_194 = xi_164 + xi_190 - xi_191; - const double xi_195 = xi_176 - xi_177; - const double xi_196 = xi_173 - xi_175 + xi_195; - const double xi_197 = xi_146 + xi_195; - const double forceTerm_0 = - xi_31 * -1.5 - xi_34 * xi_36 - xi_36 * xi_38 - xi_36 * xi_39; - const double forceTerm_1 = xi_40 + xi_47; - const double forceTerm_2 = -xi_40 + xi_47; - const double forceTerm_3 = -xi_48 + xi_51; - const double forceTerm_4 = xi_48 + xi_51; - const double forceTerm_5 = xi_52 + xi_53; - const double forceTerm_6 = -xi_52 + xi_53; - const double forceTerm_7 = xi_58 + xi_60 + xi_67; - const double forceTerm_8 = xi_57 + xi_67 + xi_68; - const double forceTerm_9 = xi_57 + xi_60 + xi_70; - const double forceTerm_10 = xi_58 + xi_68 + xi_70; - const double forceTerm_11 = xi_66 + xi_71 + xi_75; - const double forceTerm_12 = xi_69 + xi_75 + xi_76; - const double forceTerm_13 = xi_60 + xi_78 + xi_80; - const double forceTerm_14 = xi_68 + xi_77 + xi_80; - const double forceTerm_15 = xi_66 + xi_76 + xi_82; - const double forceTerm_16 = xi_69 + xi_71 + xi_82; - const double forceTerm_17 = xi_60 + xi_77 + xi_83; - const double forceTerm_18 = xi_68 + xi_78 + xi_83; - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0] = - forceTerm_0 + xi_103 * 0.0238095238095238 + xi_202 + xi_86 * 0.1 + - xi_89 * 0.0428571428571429 + xi_99 * -0.5; - _data_pdfs_20_31_10[_stride_pdfs_0 * ctr_0] = - forceTerm_1 - xi_107 + xi_114 + xi_119 + xi_128 + xi_200; - _data_pdfs_20_32_10[_stride_pdfs_0 * ctr_0] = - forceTerm_2 + xi_107 + xi_113 - xi_119 + xi_128 + xi_219; - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0] = - forceTerm_3 - xi_131 + xi_134 + xi_136 + xi_137 + xi_207; - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0] = - forceTerm_4 + xi_131 - xi_134 + xi_137 + xi_138 + xi_203; - _data_pdfs_20_35_10[_stride_pdfs_0 * ctr_0] = - forceTerm_5 - xi_141 + xi_143 + xi_146 + xi_147 + xi_209; - _data_pdfs_20_36_10[_stride_pdfs_0 * ctr_0] = - forceTerm_6 + xi_141 - xi_143 + xi_147 + xi_148 + xi_210; - _data_pdfs_20_37_10[_stride_pdfs_0 * ctr_0] = - forceTerm_7 + xi_153 + xi_157 + xi_162 + xi_204; - _data_pdfs_20_38_10[_stride_pdfs_0 * ctr_0] = - forceTerm_8 + xi_157 + xi_163 + xi_165 + xi_213; - _data_pdfs_20_39_10[_stride_pdfs_0 * ctr_0] = - forceTerm_9 + xi_162 + xi_163 + xi_167 + xi_208; - _data_pdfs_20_310_10[_stride_pdfs_0 * ctr_0] = - forceTerm_10 + xi_153 + xi_165 + xi_167 + xi_217; - _data_pdfs_20_311_10[_stride_pdfs_0 * ctr_0] = - forceTerm_11 + xi_171 + xi_179 + xi_182 + xi_214; - _data_pdfs_20_312_10[_stride_pdfs_0 * ctr_0] = - forceTerm_12 + xi_179 + xi_183 + xi_184 + xi_198; - _data_pdfs_20_313_10[_stride_pdfs_0 * ctr_0] = - forceTerm_13 + xi_188 + xi_189 + xi_192 + xi_201; - _data_pdfs_20_314_10[_stride_pdfs_0 * ctr_0] = - forceTerm_14 + xi_189 + xi_193 + xi_194 + xi_206; - _data_pdfs_20_315_10[_stride_pdfs_0 * ctr_0] = - forceTerm_15 + xi_182 + xi_184 + xi_196 + xi_215; - _data_pdfs_20_316_10[_stride_pdfs_0 * ctr_0] = - forceTerm_16 + xi_171 + xi_183 + xi_196 + xi_199; - _data_pdfs_20_317_10[_stride_pdfs_0 * ctr_0] = - forceTerm_17 + xi_192 + xi_193 + xi_197 + xi_216; - _data_pdfs_20_318_10[_stride_pdfs_0 * ctr_0] = - forceTerm_18 + xi_188 + xi_194 + xi_197 + xi_212; +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_0, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_force_0; ctr_0 += 1) + { + const double xi_207 = _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0]; + const double xi_208 = _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0]; + const double xi_209 = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0]; + const double xi_210 = _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0]; + const double xi_211 = _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0]; + const double xi_212 = _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0]; + const double xi_213 = _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0]; + const double xi_214 = _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0]; + const double xi_215 = _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0]; + const double xi_216 = _data_force_20_30_10[_stride_force_0*ctr_0]; + const double xi_217 = _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0]; + const double xi_218 = _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0]; + const double xi_219 = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + const double xi_220 = _data_force_20_31_10[_stride_force_0*ctr_0]; + const double xi_221 = _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0]; + const double xi_222 = _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0]; + const double xi_223 = _data_force_20_32_10[_stride_force_0*ctr_0]; + const double xi_224 = _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0]; + const double xi_225 = _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0]; + const double xi_226 = _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0]; + const double xi_227 = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0]; + const double xi_228 = _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0]; + const double xi_0 = xi_211 + xi_221; + const double xi_1 = xi_208 + xi_214; + const double xi_2 = xi_0 + xi_1 + xi_209; + const double xi_3 = xi_213 + xi_218; + const double xi_4 = xi_224 + xi_3; + const double xi_5 = xi_228 + xi_4; + const double xi_6 = xi_222 + xi_225; + const double xi_7 = xi_215 + xi_6; + const double xi_8 = xi_217 + xi_227; + const double xi_9 = xi_207 + xi_210; + const double xi_10 = xi_212 + xi_226; + const double xi_12 = -xi_226; + const double xi_13 = -xi_227; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -xi_215; + const double xi_16 = -xi_228; + const double xi_17 = -xi_217; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -xi_221; + const double xi_21 = xi_20 + xi_211; + const double xi_22 = -xi_225; + const double xi_23 = -xi_207; + const double xi_24 = -xi_210; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -xi_214; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -xi_218; + const double xi_29 = -xi_212; + const double xi_30 = xi_213 + xi_24 + xi_28 + xi_29; + const double xi_31 = xi_215 + xi_226; + const double xi_32 = xi_228 + xi_31 + xi_8; + const double xi_33 = xi_16 + xi_21 + xi_217; + const double xi_34 = xi_226 + xi_26; + const double xi_35 = xi_15 + xi_208; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_217 + xi_225 + xi_9; + const double xi_38 = xi_210 + xi_28; + const double xi_39 = xi_213 + xi_22 + xi_38; + const double xi_55 = xi_220*0.166666666666667; + const double xi_63 = xi_216*0.166666666666667; + const double xi_67 = xi_223*0.166666666666667; + const double xi_70 = xi_220*0.5; + const double xi_74 = xi_216*0.0833333333333333; + const double xi_78 = xi_220*0.0833333333333333; + const double xi_88 = xi_223*0.0833333333333333; + const double xi_99 = -xi_219; + const double xi_100 = xi_212*3.0 + xi_222*3.0 + xi_99; + const double xi_101 = omega_even*(xi_100 + xi_207*3.0 + xi_210*-3.0 + xi_213*-3.0 + xi_218*-3.0 + xi_224*3.0 + xi_225*-3.0); + const double xi_102 = xi_210*2.0 + xi_213*2.0 + xi_218*2.0 + xi_225*2.0; + const double xi_103 = xi_102 + xi_209*5.0 + xi_227*5.0; + const double xi_104 = omega_even*(xi_100 + xi_103 + xi_207*-2.0 + xi_208*-5.0 + xi_214*-5.0 + xi_215*-5.0 + xi_224*-2.0 + xi_226*-5.0); + const double xi_107 = -xi_213; + const double xi_108 = xi_107 + xi_22; + const double xi_109 = -xi_211; + const double xi_112 = -xi_208; + const double xi_113 = xi_112 + xi_19 + xi_27; + const double xi_115 = xi_215*2.0; + const double xi_116 = xi_208*2.0; + const double xi_117 = xi_214*2.0 + xi_226*2.0; + const double xi_118 = omega_even*(xi_103 + xi_115 + xi_116 + xi_117 + xi_207*5.0 + xi_211*-7.0 + xi_212*-4.0 + xi_217*-7.0 + xi_221*-7.0 + xi_222*-4.0 + xi_224*5.0 + xi_228*-7.0 + xi_99); + const double xi_119 = xi_107 + xi_224 + xi_225 + xi_23 + xi_38; + const double xi_121 = xi_119*xi_120; + const double xi_122 = xi_228*2.0; + const double xi_123 = xi_221*2.0; + const double xi_124 = xi_211*-2.0 + xi_217*2.0; + const double xi_125 = -xi_122 + xi_123 + xi_124 + xi_25 + xi_4; + const double xi_127 = xi_125*xi_126; + const double xi_128 = -xi_127; + const double xi_133 = xi_118*-0.0198412698412698; + const double xi_144 = xi_112 + xi_13 + xi_209 + xi_215 + xi_34; + const double xi_145 = xi_120*xi_144; + const double xi_146 = xi_122 - xi_123 + xi_124 + xi_14 + xi_209 + xi_214 + xi_35; + const double xi_147 = xi_126*xi_146; + const double xi_149 = -xi_147; + const double xi_150 = xi_210 + xi_218; + const double xi_151 = xi_108 + xi_150 + xi_222 + xi_29; + const double xi_152 = xi_120*xi_151; + const double xi_153 = -xi_115 - xi_116 + xi_117 + xi_30 + xi_6; + const double xi_154 = xi_126*xi_153; + const double xi_155 = -xi_154; + const double xi_162 = xi_118*0.0138888888888889; + const double xi_177 = xi_104*-0.00714285714285714; + const double xi_179 = xi_101*0.025; + const double xi_182 = xi_153*xi_181; + const double xi_184 = xi_151*xi_183; + const double xi_185 = xi_118*-0.00396825396825397; + const double xi_189 = xi_125*xi_181; + const double xi_190 = xi_119*xi_183; + const double xi_196 = xi_104*0.0178571428571429; + const double xi_199 = xi_144*xi_183; + const double xi_200 = xi_146*xi_181; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_219 + xi_8 + xi_9; + const double xi_11 = 1 / (rho); + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_40 = u_0*xi_216; + const double xi_41 = xi_40*0.333333333333333; + const double xi_47 = -xi_41; + const double xi_105 = rho*(u_0*u_0); + const double xi_141 = rho*u_0; + const double xi_142 = -vel0Term + xi_141 + xi_32; + const double xi_143 = xi_131*xi_142; + const double xi_167 = xi_142*xi_164; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_42 = u_1*xi_220; + const double xi_43 = xi_42*0.333333333333333; + const double xi_48 = -xi_43; + const double xi_69 = u_1*0.5; + const double xi_72 = xi_71*(u_0*xi_70 + xi_216*xi_69); + const double xi_73 = -xi_72; + const double xi_110 = rho*(u_1*u_1); + const double xi_111 = xi_109 + xi_110 + xi_20; + const double xi_129 = rho*u_1; + const double xi_130 = -vel1Term + xi_109 + xi_129 + xi_221 + xi_37; + const double xi_132 = xi_130*xi_131; + const double xi_165 = xi_130*xi_164; + const double xi_166 = xi_127 + xi_165; + const double xi_175 = -xi_165; + const double xi_176 = xi_128 + xi_175; + const double xi_191 = xi_165 - xi_189 + xi_190; + const double xi_192 = xi_175 + xi_189 - xi_190; + const double u_2 = xi_11*(vel2Term + xi_208 + xi_27 + xi_30); + const double xi_44 = u_2*xi_223; + const double xi_45 = xi_44*0.333333333333333; + const double xi_46 = (-omega_bulk + 2.0)*(xi_41 + xi_43 + xi_45); + const double xi_49 = xi_44*0.666666666666667 + xi_47 + xi_48; + const double xi_52 = -xi_45; + const double xi_53 = xi_42*0.666666666666667 + xi_47 + xi_52; + const double xi_54 = xi_40*0.666666666666667 + xi_48 + xi_52; + const double xi_57 = xi_49*xi_56; + const double xi_58 = -xi_57; + const double xi_59 = xi_54*xi_56; + const double xi_60 = -xi_59; + const double xi_62 = xi_53*xi_61 + xi_58 + xi_60; + const double xi_64 = xi_53*xi_56; + const double xi_65 = -xi_64; + const double xi_66 = xi_54*xi_61 + xi_58 + xi_65; + const double xi_68 = xi_49*xi_61 + xi_60 + xi_65; + const double xi_75 = xi_59 - xi_74; + const double xi_77 = -xi_49*xi_76; + const double xi_79 = xi_46*0.125; + const double xi_80 = xi_64 + xi_79; + const double xi_81 = xi_78 + xi_80; + const double xi_82 = xi_77 + xi_81; + const double xi_83 = xi_59 + xi_74; + const double xi_84 = -xi_78 + xi_80; + const double xi_85 = xi_77 + xi_84; + const double xi_86 = xi_71*(u_2*xi_70 + xi_223*xi_69); + const double xi_87 = -xi_54*xi_76; + const double xi_89 = xi_57 + xi_88; + const double xi_90 = xi_87 + xi_89; + const double xi_91 = -xi_86; + const double xi_92 = xi_71*(u_0*xi_223*0.5 + u_2*xi_216*0.5); + const double xi_93 = -xi_92; + const double xi_94 = -xi_53*xi_76; + const double xi_95 = xi_79 + xi_89 + xi_94; + const double xi_96 = xi_57 - xi_88; + const double xi_97 = xi_87 + xi_96; + const double xi_98 = xi_79 + xi_94 + xi_96; + const double xi_106 = rho*(u_2*u_2); + const double xi_114 = omega_bulk*(xi_105 + xi_106 + xi_108 + xi_111 + xi_113 + xi_219 + xi_24 + xi_28); + const double xi_134 = -xi_106 + xi_212 + xi_222; + const double xi_135 = omega_shear*(xi_1 + xi_111 + xi_134 + xi_18 - xi_224 + xi_23 + xi_31); + const double xi_136 = xi_135*0.125; + const double xi_137 = omega_shear*(xi_102 + xi_105*2.0 + xi_109 - xi_110 + xi_113 + xi_134 + xi_20 + xi_207 + xi_209*-2.0 + xi_224 + xi_227*-2.0); + const double xi_138 = xi_137*-0.0416666666666667; + const double xi_139 = xi_101*-0.05 + xi_138; + const double xi_140 = xi_104*0.0142857142857143 + xi_133 + xi_136 + xi_139; + const double xi_148 = xi_104*-0.0357142857142857 + xi_133 + xi_137*0.0833333333333333; + const double xi_156 = rho*u_2 - vel2Term + xi_10 + xi_107 + xi_112 + xi_150 + xi_214; + const double xi_157 = xi_131*xi_156; + const double xi_158 = xi_104*-0.0214285714285714 + xi_118*0.0158730158730159 - xi_136 + xi_139; + const double xi_161 = xi_135*0.0625; + const double xi_168 = xi_114*0.0416666666666667; + const double xi_169 = xi_137*0.0208333333333333 + xi_168; + const double xi_170 = -xi_167 + xi_169; + const double xi_171 = xi_149 + xi_170; + const double xi_173 = xi_167 + xi_169; + const double xi_174 = xi_147 + xi_173; + const double xi_186 = xi_156*xi_164; + const double xi_187 = xi_185 + xi_186; + const double xi_188 = -xi_182 + xi_184 + xi_187; + const double xi_195 = -xi_161; + const double xi_198 = xi_154 + xi_187; + const double xi_201 = xi_170 - xi_199 + xi_200; + const double xi_203 = xi_173 + xi_199 - xi_200; + const double xi_204 = xi_185 - xi_186; + const double xi_205 = xi_182 - xi_184 + xi_204; + const double xi_206 = xi_155 + xi_204; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_160 = xi_159*(-p_1 + u_0*xi_129); + const double xi_163 = -xi_160 + xi_161 + xi_162; + const double xi_172 = xi_160 + xi_161 + xi_162; + const double p_2 = xi_36; + const double xi_194 = xi_159*(-p_2 + u_2*xi_141); + const double xi_197 = -xi_194 + xi_195 + xi_196; + const double xi_202 = xi_194 + xi_195 + xi_196; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_178 = xi_159*(-p_5 + u_2*xi_129); + const double xi_180 = xi_138 + xi_168 + xi_177 + xi_178 + xi_179; + const double xi_193 = xi_138 + xi_168 + xi_177 - xi_178 + xi_179; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_210 + xi_3 + xi_7; + const double forceTerm_0 = xi_46*-1.5 - xi_49*xi_51 - xi_51*xi_53 - xi_51*xi_54; + const double forceTerm_1 = xi_55 + xi_62; + const double forceTerm_2 = -xi_55 + xi_62; + const double forceTerm_3 = -xi_63 + xi_66; + const double forceTerm_4 = xi_63 + xi_66; + const double forceTerm_5 = xi_67 + xi_68; + const double forceTerm_6 = -xi_67 + xi_68; + const double forceTerm_7 = xi_73 + xi_75 + xi_82; + const double forceTerm_8 = xi_72 + xi_82 + xi_83; + const double forceTerm_9 = xi_72 + xi_75 + xi_85; + const double forceTerm_10 = xi_73 + xi_83 + xi_85; + const double forceTerm_11 = xi_81 + xi_86 + xi_90; + const double forceTerm_12 = xi_84 + xi_90 + xi_91; + const double forceTerm_13 = xi_75 + xi_93 + xi_95; + const double forceTerm_14 = xi_83 + xi_92 + xi_95; + const double forceTerm_15 = xi_81 + xi_91 + xi_97; + const double forceTerm_16 = xi_84 + xi_86 + xi_97; + const double forceTerm_17 = xi_75 + xi_92 + xi_98; + const double forceTerm_18 = xi_83 + xi_93 + xi_98; + _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0] = forceTerm_0 + xi_101*0.1 + xi_104*0.0428571428571429 + xi_114*-0.5 + xi_118*0.0238095238095238 + xi_219; + _data_pdfs_20_31_10[_stride_pdfs_0*ctr_0] = forceTerm_1 - xi_121 + xi_128 + xi_132 + xi_140 + xi_224; + _data_pdfs_20_32_10[_stride_pdfs_0*ctr_0] = forceTerm_2 + xi_121 + xi_127 - xi_132 + xi_140 + xi_207; + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0] = forceTerm_3 - xi_143 + xi_145 + xi_147 + xi_148 + xi_227; + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0] = forceTerm_4 + xi_143 - xi_145 + xi_148 + xi_149 + xi_209; + _data_pdfs_20_35_10[_stride_pdfs_0*ctr_0] = forceTerm_5 - xi_152 + xi_155 + xi_157 + xi_158 + xi_222; + _data_pdfs_20_36_10[_stride_pdfs_0*ctr_0] = forceTerm_6 + xi_152 + xi_154 - xi_157 + xi_158 + xi_212; + _data_pdfs_20_37_10[_stride_pdfs_0*ctr_0] = forceTerm_7 + xi_163 + xi_166 + xi_171 + xi_228; + _data_pdfs_20_38_10[_stride_pdfs_0*ctr_0] = forceTerm_8 + xi_166 + xi_172 + xi_174 + xi_211; + _data_pdfs_20_39_10[_stride_pdfs_0*ctr_0] = forceTerm_9 + xi_171 + xi_172 + xi_176 + xi_217; + _data_pdfs_20_310_10[_stride_pdfs_0*ctr_0] = forceTerm_10 + xi_163 + xi_174 + xi_176 + xi_221; + _data_pdfs_20_311_10[_stride_pdfs_0*ctr_0] = forceTerm_11 + xi_180 + xi_188 + xi_191 + xi_213; + _data_pdfs_20_312_10[_stride_pdfs_0*ctr_0] = forceTerm_12 + xi_188 + xi_192 + xi_193 + xi_225; + _data_pdfs_20_313_10[_stride_pdfs_0*ctr_0] = forceTerm_13 + xi_197 + xi_198 + xi_201 + xi_215; + _data_pdfs_20_314_10[_stride_pdfs_0*ctr_0] = forceTerm_14 + xi_198 + xi_202 + xi_203 + xi_208; + _data_pdfs_20_315_10[_stride_pdfs_0*ctr_0] = forceTerm_15 + xi_191 + xi_193 + xi_205 + xi_218; + _data_pdfs_20_316_10[_stride_pdfs_0*ctr_0] = forceTerm_16 + xi_180 + xi_192 + xi_205 + xi_210; + _data_pdfs_20_317_10[_stride_pdfs_0*ctr_0] = forceTerm_17 + xi_201 + xi_202 + xi_206 + xi_226; + _data_pdfs_20_318_10[_stride_pdfs_0*ctr_0] = forceTerm_18 + xi_197 + xi_203 + xi_206 + xi_214; + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void kernel_stream( - double *RESTRICT const _data_pdfs, double *RESTRICT _data_pdfs_tmp, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) { - _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_30_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_31_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_32_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_35_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_36_10[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_37_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_38_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_39_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_20_310_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_311_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_312_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_313_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_2m1_314_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_315_1m1[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_316_11[_stride_pdfs_0 * ctr_0]; - _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_317_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0 * ctr_0] = - _data_pdfs_21_318_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_0, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + for (int64_t ctr_0 = 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) + { + _data_pdfs_tmp_20_30_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_30_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_31_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_31_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_32_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_32_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_33_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_34_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_35_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_35_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_36_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_36_10[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_37_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_37_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_38_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_38_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_39_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_39_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_310_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_20_310_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_311_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_311_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_312_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_312_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_313_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_313_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_314_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_2m1_314_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_pdfs_tmp_20_315_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_315_1m1[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_316_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_316_11[_stride_pdfs_0*ctr_0]; + _data_pdfs_tmp_20_317_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_317_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_pdfs_tmp_20_318_10[_stride_pdfs_tmp_0*ctr_0] = _data_pdfs_21_318_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_kernel_stream -const real_t MRTLatticeModel::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t MRTLatticeModel::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; -void MRTLatticeModel::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +const real_t MRTLatticeModel::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t MRTLatticeModel::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; + +void MRTLatticeModel::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, - _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, - _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, - omega_shear); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & omega_even = lm.omega_even_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & omega_odd = lm.omega_odd_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, omega_shear); + pdfs->swapDataPointers(pdfs_tmp); + } -void MRTLatticeModel::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void MRTLatticeModel::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_0 = int64_t(force->xStride()); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - omega_bulk, omega_even, omega_odd, omega_shear); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & omega_even = lm.omega_even_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & omega_odd = lm.omega_odd_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_0 = int64_t(force->xStride()); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_0, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, omega_bulk, omega_even, omega_odd, omega_shear); } -void MRTLatticeModel::Sweep::stream(IBlock *block, - const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void MRTLatticeModel::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, - _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, - _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_0 = int64_t(pdfs_tmp->xStride()); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_0, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer &operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::MRTLatticeModel &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::MRTLatticeModel & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer &operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::MRTLatticeModel &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::MRTLatticeModel & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModel.h b/src/walberla_bridge/generated_kernels/MRTLatticeModel.h index 3b3221b0e11..1ec39c7b520 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModel.h +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModel.h @@ -687,16 +687,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModel & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const MRTLatticeModel & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModel & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const MRTLatticeModel & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp index 62dcc9ed0f0..5e2d85b605f 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.cpp @@ -18,3624 +18,1558 @@ #include -#include "MRTLatticeModelAvx.h" #include "core/DataTypes.h" #include "core/Macros.h" #include "lbm/field/PdfField.h" #include "lbm/sweeps/Streaming.h" +#include "MRTLatticeModelAvx.h" #ifdef _MSC_VER -#pragma warning(disable : 4458) +# pragma warning( disable : 4458 ) #endif #define FUNC_PREFIX -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-parameter" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" +# pragma GCC diagnostic ignored "-Wunused-parameter" #endif + #include + + using namespace std; namespace walberla { namespace lbm { namespace internal_kernel_streamCollide { -static FUNC_PREFIX void kernel_streamCollide( - double *RESTRICT const _data_force, double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, - int64_t const _size_force_1, int64_t const _size_force_2, - int64_t const _stride_force_1, int64_t const _stride_force_2, - int64_t const _stride_force_3, int64_t const _stride_pdfs_1, - int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, - int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3, double omega_bulk, double omega_even, - double omega_odd, double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; +static FUNC_PREFIX void kernel_streamCollide(double * RESTRICT const _data_force, double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 1; ctr_2 < _size_force_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_force_1 - 1; ctr_1 += 1) { - for (int64_t ctr_0 = 1; - ctr_0 < (int64_t)((_size_force_0 - 2) / (4)) * (4) + 1; - ctr_0 += 4) { - const __m256d xi_0 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_1 = _mm256_add_pd( - xi_0, _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])); - const __m256d xi_2 = _mm256_add_pd( - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_20_31_1m1[ctr_0]), - _mm256_loadu_pd(&_data_pdfs_21_315_1m1[ctr_0])), - _mm256_loadu_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_3 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_2m1_312_11[ctr_0]), - _mm256_loadu_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_4 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1]), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - const __m256d xi_5 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_20_32_11[ctr_0]), - _mm256_loadu_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_6 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]), - _mm256_loadu_pd(&_data_pdfs_21_36_10[ctr_0])); - const __m256d xi_8 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - const __m256d xi_9 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])), - xi_8); - const __m256d xi_10 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - const __m256d xi_11 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_12 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])); - const __m256d xi_13 = - _mm256_add_pd(_mm256_add_pd(xi_10, xi_11), xi_12); - const __m256d xi_14 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_32_11[ctr_0])); - const __m256d xi_15 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_16 = _mm256_add_pd(xi_14, xi_15); - const __m256d xi_17 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_18 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_19 = _mm256_add_pd(xi_17, xi_18); - const __m256d xi_20 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); - const __m256d xi_21 = _mm256_add_pd(xi_10, xi_20); - const __m256d xi_22 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_315_1m1[ctr_0])); - const __m256d xi_23 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_21_36_10[ctr_0])); - const __m256d xi_24 = - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_17, xi_22), xi_23), - _mm256_loadu_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_40 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_loadu_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_48 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_loadu_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_52 = - _mm256_mul_pd(_mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667), - _mm256_loadu_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_55 = - _mm256_mul_pd(_mm256_set_pd(0.5, 0.5, 0.5, 0.5), - _mm256_loadu_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_59 = _mm256_mul_pd( - _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_loadu_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_63 = _mm256_mul_pd( - _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_loadu_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_73 = _mm256_mul_pd( - _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333), - _mm256_loadu_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_84 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_30_10[ctr_0])); - const __m256d xi_85 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_loadu_pd(&_data_pdfs_21_36_10[ctr_0])), - _mm256_mul_pd(_mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_loadu_pd(&_data_pdfs_2m1_35_10[ctr_0]))), - xi_84); - const __m256d xi_86 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_loadu_pd( - &_data_pdfs_21_315_1m1[ctr_0])), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_loadu_pd( - &_data_pdfs_21_316_11[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_311_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_312_11[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_loadu_pd(&_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(3.0, 3.0, 3.0, 3.0), - _mm256_loadu_pd(&_data_pdfs_20_32_11[ctr_0]))), - xi_85), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_87 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_315_1m1[ctr_0])), - _mm256_mul_pd( - _mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_316_11[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_311_1m1[ctr_0]))), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_312_11[ctr_0]))); - const __m256d xi_88 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1]))), - xi_87); - const __m256d xi_89 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, - -5.0), - _mm256_loadu_pd( - &_data_pdfs_21_317_10[ctr_0 + - 1])), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, - -5.0), - _mm256_loadu_pd( - &_data_pdfs_21_318_10[ctr_0 - - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_313_10[ctr_0 + - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_314_10[ctr_0 - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_loadu_pd( - &_data_pdfs_20_31_1m1[ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_loadu_pd(&_data_pdfs_20_32_11[ctr_0]))), - xi_85), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_92 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - const __m256d xi_93 = _mm256_add_pd(xi_18, xi_92); - const __m256d xi_94 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - const __m256d xi_97 = - _mm256_mul_pd(_mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_98 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_11, xi_15), xi_21), xi_97); - const __m256d xi_100 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_101 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - const __m256d xi_102 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]))); - const __m256d xi_103 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd( - -7.0, - -7.0, - -7.0, - -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_310_11 - [ctr_0 - - 1])), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, - -7.0, - -7.0, - -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_37_1m1 - [ctr_0 + - 1]))), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_38_1m1 - [ctr_0 - - 1]))), - _mm256_mul_pd( - _mm256_set_pd( - -7.0, -7.0, -7.0, - -7.0), - _mm256_loadu_pd( - &_data_pdfs_20_39_11 - [ctr_0 + - 1]))), - _mm256_mul_pd( - _mm256_set_pd(-4.0, -4.0, - -4.0, -4.0), - _mm256_loadu_pd( - &_data_pdfs_21_36_10 - [ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(-4.0, -4.0, - -4.0, -4.0), - _mm256_loadu_pd( - &_data_pdfs_2m1_35_10 - [ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd( - &_data_pdfs_20_31_1m1 - [ctr_0]))), - _mm256_mul_pd( - _mm256_set_pd(5.0, 5.0, 5.0, 5.0), - _mm256_loadu_pd( - &_data_pdfs_20_32_11[ctr_0]))), - xi_100), - xi_101), - xi_102), - xi_84), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_104 = _mm256_add_pd( - xi_92, _mm256_loadu_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_105 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_104, xi_14), xi_22), - _mm256_loadu_pd(&_data_pdfs_20_31_1m1[ctr_0])), - _mm256_loadu_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_107 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_108 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d xi_109 = - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_110 = _mm256_add_pd( - _mm256_mul_pd(_mm256_set_pd(-2.0, -2.0, -2.0, -2.0), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])), - _mm256_mul_pd(_mm256_set_pd(2.0, 2.0, 2.0, 2.0), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1]))); - const __m256d xi_111 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_108, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - xi_109), - xi_110), - xi_14), - xi_19), - xi_2); - const __m256d xi_113 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_114 = - _mm256_mul_pd(xi_113, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_116 = _mm256_add_pd( - xi_94, _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - const __m256d xi_120 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_124 = _mm256_mul_pd( - xi_103, _mm256_set_pd(-0.0198412698412698, -0.0198412698412698, - -0.0198412698412698, -0.0198412698412698)); - const __m256d xi_132 = _mm256_add_pd( - xi_97, _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d xi_133 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_12, xi_132), xi_20), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - const __m256d xi_134 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_135 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_109, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_1), - xi_108), - xi_110), - xi_13); - const __m256d xi_136 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_138 = - _mm256_mul_pd(xi_136, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_139 = - _mm256_add_pd(_mm256_loadu_pd(&_data_pdfs_21_315_1m1[ctr_0]), - _mm256_loadu_pd(&_data_pdfs_21_316_11[ctr_0])); - const __m256d xi_140 = - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_139, xi_23), xi_93), - _mm256_loadu_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_141 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_144 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_100, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_101, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_102), - xi_24), - xi_3); - const __m256d xi_145 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_146 = - _mm256_mul_pd(xi_145, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_148 = xi_145; - const __m256d xi_152 = _mm256_mul_pd( - xi_103, _mm256_set_pd(0.0138888888888889, 0.0138888888888889, - 0.0138888888888889, 0.0138888888888889)); - const __m256d xi_168 = _mm256_mul_pd( - xi_89, _mm256_set_pd(-0.00714285714285714, -0.00714285714285714, - -0.00714285714285714, -0.00714285714285714)); - const __m256d xi_170 = - _mm256_mul_pd(xi_86, _mm256_set_pd(0.025, 0.025, 0.025, 0.025)); - const __m256d xi_173 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_175 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_176 = _mm256_mul_pd( - xi_103, - _mm256_set_pd(-0.00396825396825397, -0.00396825396825397, - -0.00396825396825397, -0.00396825396825397)); - const __m256d xi_180 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_181 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_187 = _mm256_mul_pd( - xi_89, _mm256_set_pd(0.0178571428571429, 0.0178571428571429, - 0.0178571428571429, 0.0178571428571429)); - const __m256d xi_190 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_191 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d vel0Term = _mm256_add_pd( - _mm256_add_pd(xi_1, - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - const __m256d vel1Term = _mm256_add_pd( - xi_2, _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d vel2Term = _mm256_add_pd( - xi_3, _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - const __m256d rho = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel0Term, vel1Term), - vel2Term), - xi_4), - xi_5), - xi_6), - _mm256_loadu_pd(&_data_pdfs_20_30_10[ctr_0])); - const __m256d xi_7 = - _mm256_div_pd(_mm256_set_pd(1.0, 1.0, 1.0, 1.0), rho); - const __m256d u_0 = _mm256_mul_pd( - xi_7, _mm256_add_pd(_mm256_add_pd(vel0Term, xi_13), xi_9)); - const __m256d xi_25 = - _mm256_mul_pd(u_0, _mm256_loadu_pd(&_data_force_20_30_10[ctr_0])); - const __m256d xi_26 = _mm256_mul_pd( - xi_25, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_32 = - _mm256_mul_pd(xi_26, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_90 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_0, u_0))); - const __m256d xi_129 = _mm256_mul_pd(rho, u_0); - const __m256d xi_130 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel0Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_120), - xi_129), - xi_4), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - const __m256d xi_131 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_158 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d u_1 = _mm256_mul_pd( - xi_7, - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel1Term, xi_16), xi_19), - xi_8), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1]))); - const __m256d xi_27 = - _mm256_mul_pd(u_1, _mm256_loadu_pd(&_data_force_20_31_10[ctr_0])); - const __m256d xi_28 = _mm256_mul_pd( - xi_27, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_33 = - _mm256_mul_pd(xi_28, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_54 = - _mm256_mul_pd(u_1, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_57 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd(u_0, xi_55), - _mm256_mul_pd(xi_54, - _mm256_loadu_pd(&_data_force_20_30_10[ctr_0]))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_58 = - _mm256_mul_pd(xi_57, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_95 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_1, u_1))); - const __m256d xi_96 = - _mm256_add_pd(_mm256_add_pd(xi_9, xi_94), xi_95); - const __m256d xi_115 = _mm256_mul_pd(rho, u_1); - const __m256d xi_117 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - vel1Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_115), - xi_116), - xi_5), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])), - _mm256_loadu_pd(&_data_pdfs_2m1_312_11[ctr_0])); - const __m256d xi_119 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_150 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(u_0, xi_115), xi_116), xi_8), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_155 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_156 = xi_155; - const __m256d xi_157 = _mm256_add_pd(xi_113, xi_156); - const __m256d xi_166 = - _mm256_mul_pd(xi_155, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_167 = _mm256_add_pd(xi_114, xi_166); - const __m256d xi_182 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_180, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_156), - xi_181); - const __m256d xi_183 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_181, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_166), - xi_180); - const __m256d u_2 = _mm256_mul_pd( - xi_7, _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel2Term, xi_21), xi_24), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1]))); - const __m256d xi_29 = - _mm256_mul_pd(u_2, _mm256_loadu_pd(&_data_force_20_32_10[ctr_0])); - const __m256d xi_30 = _mm256_mul_pd( - xi_29, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_31 = _mm256_mul_pd( - _mm256_add_pd(_mm256_add_pd(xi_26, xi_28), xi_30), - _mm256_set_pd(-omega_bulk + 2.0, -omega_bulk + 2.0, - -omega_bulk + 2.0, -omega_bulk + 2.0)); - const __m256d xi_34 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_29, _mm256_set_pd(0.666666666666667, - 0.666666666666667, - 0.666666666666667, - 0.666666666666667)), - xi_32), - xi_33); - const __m256d xi_37 = - _mm256_mul_pd(xi_30, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_38 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_27, _mm256_set_pd(0.666666666666667, - 0.666666666666667, - 0.666666666666667, - 0.666666666666667)), - xi_32), - xi_37); - const __m256d xi_39 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_25, _mm256_set_pd(0.666666666666667, - 0.666666666666667, - 0.666666666666667, - 0.666666666666667)), - xi_33), - xi_37); - const __m256d xi_42 = - _mm256_mul_pd(xi_34, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_43 = - _mm256_mul_pd(xi_42, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_44 = - _mm256_mul_pd(xi_39, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_45 = - _mm256_mul_pd(xi_44, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_47 = _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(xi_38, _mm256_set_pd(xi_46, xi_46, - xi_46, xi_46)), - xi_43), - xi_45); - const __m256d xi_49 = - _mm256_mul_pd(xi_38, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_50 = - _mm256_mul_pd(xi_49, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_51 = _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(xi_39, _mm256_set_pd(xi_46, xi_46, - xi_46, xi_46)), - xi_43), - xi_50); - const __m256d xi_53 = _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(xi_34, _mm256_set_pd(xi_46, xi_46, - xi_46, xi_46)), - xi_45), - xi_50); - const __m256d xi_60 = _mm256_add_pd( - _mm256_mul_pd(xi_59, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_44); - const __m256d xi_62 = _mm256_mul_pd( - _mm256_mul_pd(xi_34, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_64 = - _mm256_mul_pd(xi_31, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_65 = _mm256_add_pd(xi_49, xi_64); - const __m256d xi_66 = _mm256_add_pd(xi_63, xi_65); - const __m256d xi_67 = _mm256_add_pd(xi_62, xi_66); - const __m256d xi_68 = _mm256_add_pd(xi_44, xi_59); - const __m256d xi_69 = _mm256_add_pd( - _mm256_mul_pd(xi_63, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_65); - const __m256d xi_70 = _mm256_add_pd(xi_62, xi_69); - const __m256d xi_71 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd(u_2, xi_55), - _mm256_mul_pd(xi_54, - _mm256_loadu_pd(&_data_force_20_32_10[ctr_0]))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_72 = _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_74 = _mm256_add_pd(xi_42, xi_73); - const __m256d xi_75 = _mm256_add_pd(xi_72, xi_74); - const __m256d xi_76 = - _mm256_mul_pd(xi_71, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_77 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_mul_pd(u_0, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_loadu_pd(&_data_force_20_32_10[ctr_0])), - _mm256_mul_pd( - _mm256_mul_pd(u_2, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_loadu_pd(&_data_force_20_30_10[ctr_0]))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_78 = - _mm256_mul_pd(xi_77, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_79 = _mm256_mul_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_80 = - _mm256_add_pd(_mm256_add_pd(xi_64, xi_74), xi_79); - const __m256d xi_81 = _mm256_add_pd( - _mm256_mul_pd(xi_73, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_42); - const __m256d xi_82 = _mm256_add_pd(xi_72, xi_81); - const __m256d xi_83 = - _mm256_add_pd(_mm256_add_pd(xi_64, xi_79), xi_81); - const __m256d xi_91 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_2, u_2))); - const __m256d xi_99 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_17, xi_22), - xi_90), - xi_91), - xi_93), - xi_96), - xi_98), - _mm256_loadu_pd(&_data_pdfs_20_30_10[ctr_0])), - _mm256_set_pd(omega_bulk, omega_bulk, omega_bulk, omega_bulk)); - const __m256d xi_121 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_91, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_loadu_pd(&_data_pdfs_21_36_10[ctr_0])), - _mm256_loadu_pd(&_data_pdfs_2m1_35_10[ctr_0])); - const __m256d xi_122 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0), - _mm256_loadu_pd( - &_data_pdfs_20_31_1m1[ctr_0])), - xi_0), - xi_120), - xi_121), - xi_16), - xi_96), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, - omega_shear)); - const __m256d xi_123 = - _mm256_mul_pd(xi_122, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_125 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_90, _mm256_set_pd( - 2.0, 2.0, - 2.0, 2.0)), - _mm256_mul_pd( - xi_95, - _mm256_set_pd( - -1.0, -1.0, -1.0, - -1.0))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, - -2.0, -2.0), - _mm256_loadu_pd( - &_data_pdfs_20_33_10 - [ctr_0 + 1]))), - _mm256_mul_pd( - _mm256_set_pd(-2.0, -2.0, - -2.0, -2.0), - _mm256_loadu_pd( - &_data_pdfs_20_34_10 - [ctr_0 - 1]))), - xi_121), - xi_87), - xi_9), - xi_94), - xi_98), - _mm256_loadu_pd(&_data_pdfs_20_31_1m1[ctr_0])), - _mm256_loadu_pd(&_data_pdfs_20_32_11[ctr_0])), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, - omega_shear)); - const __m256d xi_126 = _mm256_mul_pd( - xi_125, _mm256_set_pd(-0.0416666666666667, -0.0416666666666667, - -0.0416666666666667, -0.0416666666666667)); - const __m256d xi_127 = _mm256_add_pd( - _mm256_mul_pd(xi_86, _mm256_set_pd(-0.05, -0.05, -0.05, -0.05)), - xi_126); - const __m256d xi_128 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_89, _mm256_set_pd(0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143)), - xi_123), - xi_124), - xi_127); - const __m256d xi_137 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_125, _mm256_set_pd(0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333)), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857))), - xi_124); - const __m256d xi_142 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(rho, u_2), - _mm256_mul_pd( - vel2Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_139), - xi_6), - xi_92), - xi_97), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); - const __m256d xi_143 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_147 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_103, _mm256_set_pd(0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159)), - _mm256_mul_pd(xi_123, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714))), - xi_127); - const __m256d xi_151 = _mm256_mul_pd( - xi_122, _mm256_set_pd(0.0625, 0.0625, 0.0625, 0.0625)); - const __m256d xi_153 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_150, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_151), - xi_152); - const __m256d xi_159 = _mm256_mul_pd( - xi_99, _mm256_set_pd(0.0416666666666667, 0.0416666666666667, - 0.0416666666666667, 0.0416666666666667)); - const __m256d xi_160 = _mm256_add_pd( - _mm256_mul_pd(xi_125, _mm256_set_pd(0.0208333333333333, - 0.0208333333333333, - 0.0208333333333333, - 0.0208333333333333)), - xi_159); - const __m256d xi_161 = _mm256_add_pd( - _mm256_mul_pd(xi_158, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_160); - const __m256d xi_162 = _mm256_add_pd(xi_138, xi_161); - const __m256d xi_163 = - _mm256_add_pd(_mm256_add_pd(xi_150, xi_151), xi_152); - const __m256d xi_164 = _mm256_add_pd(xi_158, xi_160); - const __m256d xi_165 = _mm256_add_pd(xi_136, xi_164); - const __m256d xi_169 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(u_2, xi_115), xi_104), xi_17), - _mm256_loadu_pd(&_data_pdfs_21_315_1m1[ctr_0])), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_171 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_126, xi_159), xi_168), xi_169), - xi_170); - const __m256d xi_177 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_178 = _mm256_add_pd(xi_176, xi_177); - const __m256d xi_179 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_173, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_175), - xi_178); - const __m256d xi_184 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_169, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_126), - xi_159), - xi_168), - xi_170); - const __m256d xi_185 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(u_2, xi_129), xi_10), xi_132), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_186 = - _mm256_mul_pd(xi_151, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_188 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_185, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_186), - xi_187); - const __m256d xi_189 = _mm256_add_pd(xi_148, xi_178); - const __m256d xi_192 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_190, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_161), - xi_191); - const __m256d xi_193 = - _mm256_add_pd(_mm256_add_pd(xi_185, xi_186), xi_187); - const __m256d xi_194 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_191, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_164), - xi_190); - const __m256d xi_195 = _mm256_add_pd( - _mm256_mul_pd(xi_177, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_176); - const __m256d xi_196 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_175, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_173), - xi_195); - const __m256d xi_197 = _mm256_add_pd(xi_146, xi_195); - const __m256d forceTerm_0 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_31, - _mm256_set_pd(-1.5, -1.5, -1.5, -1.5)), - _mm256_mul_pd( - _mm256_mul_pd(xi_34, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd(_mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))); - const __m256d forceTerm_1 = _mm256_add_pd(xi_40, xi_47); - const __m256d forceTerm_2 = _mm256_add_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_47); - const __m256d forceTerm_3 = _mm256_add_pd( - _mm256_mul_pd(xi_48, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_51); - const __m256d forceTerm_4 = _mm256_add_pd(xi_48, xi_51); - const __m256d forceTerm_5 = _mm256_add_pd(xi_52, xi_53); - const __m256d forceTerm_6 = _mm256_add_pd( - _mm256_mul_pd(xi_52, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_53); - const __m256d forceTerm_7 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_60), xi_67); - const __m256d forceTerm_8 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_67), xi_68); - const __m256d forceTerm_9 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_60), xi_70); - const __m256d forceTerm_10 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_68), xi_70); - const __m256d forceTerm_11 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_71), xi_75); - const __m256d forceTerm_12 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_75), xi_76); - const __m256d forceTerm_13 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_78), xi_80); - const __m256d forceTerm_14 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_77), xi_80); - const __m256d forceTerm_15 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_76), xi_82); - const __m256d forceTerm_16 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_71), xi_82); - const __m256d forceTerm_17 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_77), xi_83); - const __m256d forceTerm_18 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_78), xi_83); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_30_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_103, - _mm256_set_pd(0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238)), - _mm256_mul_pd( - xi_86, - _mm256_set_pd(0.1, 0.1, 0.1, 0.1))), - _mm256_mul_pd(xi_89, - _mm256_set_pd(0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429))), - _mm256_mul_pd(xi_99, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - forceTerm_0), - _mm256_loadu_pd(&_data_pdfs_20_30_10[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_31_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_107, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_1), - xi_114), - xi_119), - xi_128), - _mm256_loadu_pd(&_data_pdfs_20_31_1m1[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_32_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_119, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_2), - xi_107), - xi_113), - xi_128), - _mm256_loadu_pd(&_data_pdfs_20_32_11[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_33_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_131, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_3), - xi_134), - xi_136), - xi_137), - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_34_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_134, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_4), - xi_131), - xi_137), - xi_138), - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_35_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_141, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_5), - xi_143), - xi_146), - xi_147), - _mm256_loadu_pd(&_data_pdfs_2m1_35_10[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_36_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_143, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_6), - xi_141), - xi_147), - xi_148), - _mm256_loadu_pd(&_data_pdfs_21_36_10[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_37_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_7, xi_153), xi_157), - xi_162), - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_38_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_8, xi_157), xi_163), - xi_165), - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_39_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_9, xi_162), xi_163), - xi_167), - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_310_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_10, xi_153), - xi_165), - xi_167), - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_311_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_11, xi_171), - xi_179), - xi_182), - _mm256_loadu_pd(&_data_pdfs_2m1_311_1m1[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_312_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_12, xi_179), - xi_183), - xi_184), - _mm256_loadu_pd(&_data_pdfs_2m1_312_11[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_313_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_13, xi_188), - xi_189), - xi_192), - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_314_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_14, xi_189), - xi_193), - xi_194), - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_315_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_15, xi_182), - xi_184), - xi_196), - _mm256_loadu_pd(&_data_pdfs_21_315_1m1[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_316_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_16, xi_171), - xi_183), - xi_196), - _mm256_loadu_pd(&_data_pdfs_21_316_11[ctr_0]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_317_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_17, xi_192), - xi_193), - xi_197), - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1]))); - _mm256_storeu_pd( - &_data_pdfs_tmp_20_318_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_18, xi_188), - xi_194), - xi_197), - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1]))); - } - for (int64_t ctr_0 = (int64_t)((_size_force_0 - 2) / (4)) * (4) + 1; - ctr_0 < _size_force_0 - 1; ctr_0 += 1) { - const double xi_0 = _data_pdfs_21_318_10[ctr_0 - 1] + - _data_pdfs_2m1_314_10[ctr_0 - 1]; - const double xi_1 = xi_0 + _data_pdfs_20_34_10[ctr_0 - 1]; - const double xi_2 = _data_pdfs_20_31_1m1[ctr_0] + - _data_pdfs_21_315_1m1[ctr_0] + - _data_pdfs_2m1_311_1m1[ctr_0]; - const double xi_3 = - _data_pdfs_2m1_312_11[ctr_0] + _data_pdfs_2m1_35_10[ctr_0]; - const double xi_4 = - _data_pdfs_20_33_10[ctr_0 + 1] + _data_pdfs_20_39_11[ctr_0 + 1]; - const double xi_5 = - _data_pdfs_20_32_11[ctr_0] + _data_pdfs_21_316_11[ctr_0]; - const double xi_6 = - _data_pdfs_21_317_10[ctr_0 + 1] + _data_pdfs_21_36_10[ctr_0]; - const double xi_8 = -_data_pdfs_20_39_11[ctr_0 + 1]; - const double xi_9 = xi_8 - _data_pdfs_20_37_1m1[ctr_0 + 1]; - const double xi_10 = -_data_pdfs_21_317_10[ctr_0 + 1]; - const double xi_11 = -_data_pdfs_2m1_313_10[ctr_0 + 1]; - const double xi_12 = -_data_pdfs_20_33_10[ctr_0 + 1]; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -_data_pdfs_20_32_11[ctr_0]; - const double xi_15 = -_data_pdfs_20_310_11[ctr_0 - 1]; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -_data_pdfs_21_316_11[ctr_0]; - const double xi_18 = -_data_pdfs_2m1_312_11[ctr_0]; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = -_data_pdfs_21_318_10[ctr_0 - 1]; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -_data_pdfs_21_315_1m1[ctr_0]; - const double xi_23 = -_data_pdfs_21_36_10[ctr_0]; - const double xi_24 = - xi_17 + xi_22 + xi_23 + _data_pdfs_2m1_311_1m1[ctr_0]; - const double xi_40 = 0.166666666666667 * _data_force_20_31_10[ctr_0]; - const double xi_48 = 0.166666666666667 * _data_force_20_30_10[ctr_0]; - const double xi_52 = 0.166666666666667 * _data_force_20_32_10[ctr_0]; - const double xi_55 = 0.5 * _data_force_20_31_10[ctr_0]; - const double xi_59 = 0.0833333333333333 * _data_force_20_30_10[ctr_0]; - const double xi_63 = 0.0833333333333333 * _data_force_20_31_10[ctr_0]; - const double xi_73 = 0.0833333333333333 * _data_force_20_32_10[ctr_0]; - const double xi_84 = -_data_pdfs_20_30_10[ctr_0]; - const double xi_85 = xi_84 + 3.0 * _data_pdfs_21_36_10[ctr_0] + - 3.0 * _data_pdfs_2m1_35_10[ctr_0]; - const double xi_86 = - omega_even * (xi_85 - 3.0 * _data_pdfs_21_315_1m1[ctr_0] - - 3.0 * _data_pdfs_21_316_11[ctr_0] - - 3.0 * _data_pdfs_2m1_311_1m1[ctr_0] - - 3.0 * _data_pdfs_2m1_312_11[ctr_0] + - 3.0 * _data_pdfs_20_31_1m1[ctr_0] + - 3.0 * _data_pdfs_20_32_11[ctr_0]); - const double xi_87 = 2.0 * _data_pdfs_21_315_1m1[ctr_0] + - 2.0 * _data_pdfs_21_316_11[ctr_0] + - 2.0 * _data_pdfs_2m1_311_1m1[ctr_0] + - 2.0 * _data_pdfs_2m1_312_11[ctr_0]; - const double xi_88 = xi_87 + 5.0 * _data_pdfs_20_33_10[ctr_0 + 1] + - 5.0 * _data_pdfs_20_34_10[ctr_0 - 1]; - const double xi_89 = - omega_even * (xi_85 + xi_88 - 2.0 * _data_pdfs_20_31_1m1[ctr_0] - - 2.0 * _data_pdfs_20_32_11[ctr_0] - - 5.0 * _data_pdfs_21_317_10[ctr_0 + 1] - - 5.0 * _data_pdfs_21_318_10[ctr_0 - 1] - - 5.0 * _data_pdfs_2m1_313_10[ctr_0 + 1] - - 5.0 * _data_pdfs_2m1_314_10[ctr_0 - 1]); - const double xi_92 = -_data_pdfs_2m1_311_1m1[ctr_0]; - const double xi_93 = xi_18 + xi_92; - const double xi_94 = -_data_pdfs_20_38_1m1[ctr_0 - 1]; - const double xi_97 = -_data_pdfs_2m1_314_10[ctr_0 - 1]; - const double xi_98 = xi_11 + xi_15 + xi_21 + xi_97; - const double xi_100 = 2.0 * _data_pdfs_2m1_313_10[ctr_0 + 1]; - const double xi_101 = 2.0 * _data_pdfs_2m1_314_10[ctr_0 - 1]; - const double xi_102 = 2.0 * _data_pdfs_21_317_10[ctr_0 + 1] + - 2.0 * _data_pdfs_21_318_10[ctr_0 - 1]; - const double xi_103 = - omega_even * (xi_100 + xi_101 + xi_102 + xi_84 + xi_88 - - 4.0 * _data_pdfs_21_36_10[ctr_0] - - 4.0 * _data_pdfs_2m1_35_10[ctr_0] - - 7.0 * _data_pdfs_20_310_11[ctr_0 - 1] - - 7.0 * _data_pdfs_20_37_1m1[ctr_0 + 1] - - 7.0 * _data_pdfs_20_38_1m1[ctr_0 - 1] - - 7.0 * _data_pdfs_20_39_11[ctr_0 + 1] + - 5.0 * _data_pdfs_20_31_1m1[ctr_0] + - 5.0 * _data_pdfs_20_32_11[ctr_0]); - const double xi_104 = xi_92 + _data_pdfs_2m1_312_11[ctr_0]; - const double xi_105 = xi_104 + xi_14 + xi_22 + - _data_pdfs_20_31_1m1[ctr_0] + - _data_pdfs_21_316_11[ctr_0]; - const double xi_107 = xi_105 * xi_106; - const double xi_108 = 2.0 * _data_pdfs_20_37_1m1[ctr_0 + 1]; - const double xi_109 = 2.0 * _data_pdfs_20_310_11[ctr_0 - 1]; - const double xi_110 = -2.0 * _data_pdfs_20_38_1m1[ctr_0 - 1] + - 2.0 * _data_pdfs_20_39_11[ctr_0 + 1]; - const double xi_111 = - -xi_108 + xi_109 + xi_110 + xi_14 + xi_19 + xi_2; - const double xi_113 = xi_111 * xi_112; - const double xi_114 = -xi_113; - const double xi_116 = xi_94 + _data_pdfs_20_310_11[ctr_0 - 1]; - const double xi_120 = _data_pdfs_21_317_10[ctr_0 + 1] + - _data_pdfs_2m1_313_10[ctr_0 + 1]; - const double xi_124 = xi_103 * -0.0198412698412698; - const double xi_132 = xi_97 + _data_pdfs_2m1_313_10[ctr_0 + 1]; - const double xi_133 = xi_12 + xi_132 + xi_20 + - _data_pdfs_20_34_10[ctr_0 - 1] + - _data_pdfs_21_317_10[ctr_0 + 1]; - const double xi_134 = xi_106 * xi_133; - const double xi_135 = xi_1 + xi_108 - xi_109 + xi_110 + xi_13; - const double xi_136 = xi_112 * xi_135; - const double xi_138 = -xi_136; - const double xi_139 = - _data_pdfs_21_315_1m1[ctr_0] + _data_pdfs_21_316_11[ctr_0]; - const double xi_140 = - xi_139 + xi_23 + xi_93 + _data_pdfs_2m1_35_10[ctr_0]; - const double xi_141 = xi_106 * xi_140; - const double xi_144 = -xi_100 - xi_101 + xi_102 + xi_24 + xi_3; - const double xi_145 = xi_112 * xi_144; - const double xi_146 = -xi_145; - const double xi_148 = xi_145; - const double xi_152 = xi_103 * 0.0138888888888889; - const double xi_168 = xi_89 * -0.00714285714285714; - const double xi_170 = xi_86 * 0.025; - const double xi_173 = xi_144 * xi_172; - const double xi_175 = xi_140 * xi_174; - const double xi_176 = xi_103 * -0.00396825396825397; - const double xi_180 = xi_111 * xi_172; - const double xi_181 = xi_105 * xi_174; - const double xi_187 = xi_89 * 0.0178571428571429; - const double xi_190 = xi_133 * xi_174; - const double xi_191 = xi_135 * xi_172; - const double vel0Term = xi_1 + _data_pdfs_20_310_11[ctr_0 - 1] + - _data_pdfs_20_38_1m1[ctr_0 - 1]; - const double vel1Term = xi_2 + _data_pdfs_20_37_1m1[ctr_0 + 1]; - const double vel2Term = xi_3 + _data_pdfs_2m1_313_10[ctr_0 + 1]; - const double rho = vel0Term + vel1Term + vel2Term + xi_4 + xi_5 + - xi_6 + _data_pdfs_20_30_10[ctr_0]; - const double xi_7 = 1 / (rho); - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_25 = u_0 * _data_force_20_30_10[ctr_0]; - const double xi_26 = xi_25 * 0.333333333333333; - const double xi_32 = -xi_26; - const double xi_90 = rho * (u_0 * u_0); - const double xi_129 = rho * u_0; - const double xi_130 = -vel0Term + xi_120 + xi_129 + xi_4 + - _data_pdfs_20_37_1m1[ctr_0 + 1]; - const double xi_131 = xi_118 * xi_130; - const double xi_158 = xi_130 * xi_154; - const double u_1 = xi_7 * (vel1Term + xi_16 + xi_19 + xi_8 + - _data_pdfs_20_38_1m1[ctr_0 - 1]); - const double xi_27 = u_1 * _data_force_20_31_10[ctr_0]; - const double xi_28 = xi_27 * 0.333333333333333; - const double xi_33 = -xi_28; - const double xi_54 = u_1 * 0.5; - const double xi_57 = - xi_56 * (u_0 * xi_55 + xi_54 * _data_force_20_30_10[ctr_0]); - const double xi_58 = -xi_57; - const double xi_95 = rho * (u_1 * u_1); - const double xi_96 = xi_9 + xi_94 + xi_95; - const double xi_115 = rho * u_1; - const double xi_117 = -vel1Term + xi_115 + xi_116 + xi_5 + - _data_pdfs_20_39_11[ctr_0 + 1] + - _data_pdfs_2m1_312_11[ctr_0]; - const double xi_119 = xi_117 * xi_118; - const double xi_150 = xi_149 * (u_0 * xi_115 + xi_116 + xi_8 + - _data_pdfs_20_37_1m1[ctr_0 + 1]); - const double xi_155 = xi_117 * xi_154; - const double xi_156 = xi_155; - const double xi_157 = xi_113 + xi_156; - const double xi_166 = -xi_155; - const double xi_167 = xi_114 + xi_166; - const double xi_182 = xi_156 - xi_180 + xi_181; - const double xi_183 = xi_166 + xi_180 - xi_181; - const double u_2 = xi_7 * (vel2Term + xi_21 + xi_24 + - _data_pdfs_2m1_314_10[ctr_0 - 1]); - const double xi_29 = u_2 * _data_force_20_32_10[ctr_0]; - const double xi_30 = xi_29 * 0.333333333333333; - const double xi_31 = (-omega_bulk + 2.0) * (xi_26 + xi_28 + xi_30); - const double xi_34 = xi_29 * 0.666666666666667 + xi_32 + xi_33; - const double xi_37 = -xi_30; - const double xi_38 = xi_27 * 0.666666666666667 + xi_32 + xi_37; - const double xi_39 = xi_25 * 0.666666666666667 + xi_33 + xi_37; - const double xi_42 = xi_34 * xi_41; - const double xi_43 = -xi_42; - const double xi_44 = xi_39 * xi_41; - const double xi_45 = -xi_44; - const double xi_47 = xi_38 * xi_46 + xi_43 + xi_45; - const double xi_49 = xi_38 * xi_41; - const double xi_50 = -xi_49; - const double xi_51 = xi_39 * xi_46 + xi_43 + xi_50; - const double xi_53 = xi_34 * xi_46 + xi_45 + xi_50; - const double xi_60 = xi_44 - xi_59; - const double xi_62 = -xi_34 * xi_61; - const double xi_64 = xi_31 * 0.125; - const double xi_65 = xi_49 + xi_64; - const double xi_66 = xi_63 + xi_65; - const double xi_67 = xi_62 + xi_66; - const double xi_68 = xi_44 + xi_59; - const double xi_69 = -xi_63 + xi_65; - const double xi_70 = xi_62 + xi_69; - const double xi_71 = - xi_56 * (u_2 * xi_55 + xi_54 * _data_force_20_32_10[ctr_0]); - const double xi_72 = -xi_39 * xi_61; - const double xi_74 = xi_42 + xi_73; - const double xi_75 = xi_72 + xi_74; - const double xi_76 = -xi_71; - const double xi_77 = - xi_56 * (u_0 * 0.5 * _data_force_20_32_10[ctr_0] + - u_2 * 0.5 * _data_force_20_30_10[ctr_0]); - const double xi_78 = -xi_77; - const double xi_79 = -xi_38 * xi_61; - const double xi_80 = xi_64 + xi_74 + xi_79; - const double xi_81 = xi_42 - xi_73; - const double xi_82 = xi_72 + xi_81; - const double xi_83 = xi_64 + xi_79 + xi_81; - const double xi_91 = rho * (u_2 * u_2); - const double xi_99 = - omega_bulk * (xi_17 + xi_22 + xi_90 + xi_91 + xi_93 + xi_96 + - xi_98 + _data_pdfs_20_30_10[ctr_0]); - const double xi_121 = - -xi_91 + _data_pdfs_21_36_10[ctr_0] + _data_pdfs_2m1_35_10[ctr_0]; - const double xi_122 = - omega_shear * (xi_0 + xi_120 + xi_121 + xi_16 + xi_96 - - _data_pdfs_20_31_1m1[ctr_0]); - const double xi_123 = xi_122 * 0.125; - const double xi_125 = - omega_shear * - (xi_121 + xi_87 + xi_9 + xi_90 * 2.0 + xi_94 - xi_95 + xi_98 - - 2.0 * _data_pdfs_20_33_10[ctr_0 + 1] - - 2.0 * _data_pdfs_20_34_10[ctr_0 - 1] + - _data_pdfs_20_31_1m1[ctr_0] + _data_pdfs_20_32_11[ctr_0]); - const double xi_126 = xi_125 * -0.0416666666666667; - const double xi_127 = xi_126 + xi_86 * -0.05; - const double xi_128 = - xi_123 + xi_124 + xi_127 + xi_89 * 0.0142857142857143; - const double xi_137 = xi_124 + xi_125 * 0.0833333333333333 + - xi_89 * -0.0357142857142857; - const double xi_142 = rho * u_2 - vel2Term + xi_139 + xi_6 + xi_92 + - xi_97 + _data_pdfs_21_318_10[ctr_0 - 1]; - const double xi_143 = xi_118 * xi_142; - const double xi_147 = xi_103 * 0.0158730158730159 - xi_123 + xi_127 + - xi_89 * -0.0214285714285714; - const double xi_151 = xi_122 * 0.0625; - const double xi_153 = -xi_150 + xi_151 + xi_152; - const double xi_159 = xi_99 * 0.0416666666666667; - const double xi_160 = xi_125 * 0.0208333333333333 + xi_159; - const double xi_161 = -xi_158 + xi_160; - const double xi_162 = xi_138 + xi_161; - const double xi_163 = xi_150 + xi_151 + xi_152; - const double xi_164 = xi_158 + xi_160; - const double xi_165 = xi_136 + xi_164; - const double xi_169 = xi_149 * (u_2 * xi_115 + xi_104 + xi_17 + - _data_pdfs_21_315_1m1[ctr_0]); - const double xi_171 = xi_126 + xi_159 + xi_168 + xi_169 + xi_170; - const double xi_177 = xi_142 * xi_154; - const double xi_178 = xi_176 + xi_177; - const double xi_179 = -xi_173 + xi_175 + xi_178; - const double xi_184 = xi_126 + xi_159 + xi_168 - xi_169 + xi_170; - const double xi_185 = xi_149 * (u_2 * xi_129 + xi_10 + xi_132 + - _data_pdfs_21_318_10[ctr_0 - 1]); - const double xi_186 = -xi_151; - const double xi_188 = -xi_185 + xi_186 + xi_187; - const double xi_189 = xi_148 + xi_178; - const double xi_192 = xi_161 - xi_190 + xi_191; - const double xi_193 = xi_185 + xi_186 + xi_187; - const double xi_194 = xi_164 + xi_190 - xi_191; - const double xi_195 = xi_176 - xi_177; - const double xi_196 = xi_173 - xi_175 + xi_195; - const double xi_197 = xi_146 + xi_195; - const double forceTerm_0 = - xi_31 * -1.5 - xi_34 * xi_36 - xi_36 * xi_38 - xi_36 * xi_39; - const double forceTerm_1 = xi_40 + xi_47; - const double forceTerm_2 = -xi_40 + xi_47; - const double forceTerm_3 = -xi_48 + xi_51; - const double forceTerm_4 = xi_48 + xi_51; - const double forceTerm_5 = xi_52 + xi_53; - const double forceTerm_6 = -xi_52 + xi_53; - const double forceTerm_7 = xi_58 + xi_60 + xi_67; - const double forceTerm_8 = xi_57 + xi_67 + xi_68; - const double forceTerm_9 = xi_57 + xi_60 + xi_70; - const double forceTerm_10 = xi_58 + xi_68 + xi_70; - const double forceTerm_11 = xi_66 + xi_71 + xi_75; - const double forceTerm_12 = xi_69 + xi_75 + xi_76; - const double forceTerm_13 = xi_60 + xi_78 + xi_80; - const double forceTerm_14 = xi_68 + xi_77 + xi_80; - const double forceTerm_15 = xi_66 + xi_76 + xi_82; - const double forceTerm_16 = xi_69 + xi_71 + xi_82; - const double forceTerm_17 = xi_60 + xi_77 + xi_83; - const double forceTerm_18 = xi_68 + xi_78 + xi_83; - _data_pdfs_tmp_20_30_10[ctr_0] = - forceTerm_0 + xi_103 * 0.0238095238095238 + xi_86 * 0.1 + - xi_89 * 0.0428571428571429 + xi_99 * -0.5 + - _data_pdfs_20_30_10[ctr_0]; - _data_pdfs_tmp_20_31_10[ctr_0] = forceTerm_1 - xi_107 + xi_114 + - xi_119 + xi_128 + - _data_pdfs_20_31_1m1[ctr_0]; - _data_pdfs_tmp_20_32_10[ctr_0] = forceTerm_2 + xi_107 + xi_113 - - xi_119 + xi_128 + - _data_pdfs_20_32_11[ctr_0]; - _data_pdfs_tmp_20_33_10[ctr_0] = forceTerm_3 - xi_131 + xi_134 + - xi_136 + xi_137 + - _data_pdfs_20_33_10[ctr_0 + 1]; - _data_pdfs_tmp_20_34_10[ctr_0] = forceTerm_4 + xi_131 - xi_134 + - xi_137 + xi_138 + - _data_pdfs_20_34_10[ctr_0 - 1]; - _data_pdfs_tmp_20_35_10[ctr_0] = forceTerm_5 - xi_141 + xi_143 + - xi_146 + xi_147 + - _data_pdfs_2m1_35_10[ctr_0]; - _data_pdfs_tmp_20_36_10[ctr_0] = forceTerm_6 + xi_141 - xi_143 + - xi_147 + xi_148 + - _data_pdfs_21_36_10[ctr_0]; - _data_pdfs_tmp_20_37_10[ctr_0] = forceTerm_7 + xi_153 + xi_157 + - xi_162 + - _data_pdfs_20_37_1m1[ctr_0 + 1]; - _data_pdfs_tmp_20_38_10[ctr_0] = forceTerm_8 + xi_157 + xi_163 + - xi_165 + - _data_pdfs_20_38_1m1[ctr_0 - 1]; - _data_pdfs_tmp_20_39_10[ctr_0] = forceTerm_9 + xi_162 + xi_163 + - xi_167 + - _data_pdfs_20_39_11[ctr_0 + 1]; - _data_pdfs_tmp_20_310_10[ctr_0] = forceTerm_10 + xi_153 + xi_165 + - xi_167 + - _data_pdfs_20_310_11[ctr_0 - 1]; - _data_pdfs_tmp_20_311_10[ctr_0] = forceTerm_11 + xi_171 + xi_179 + - xi_182 + - _data_pdfs_2m1_311_1m1[ctr_0]; - _data_pdfs_tmp_20_312_10[ctr_0] = forceTerm_12 + xi_179 + xi_183 + - xi_184 + - _data_pdfs_2m1_312_11[ctr_0]; - _data_pdfs_tmp_20_313_10[ctr_0] = forceTerm_13 + xi_188 + xi_189 + - xi_192 + - _data_pdfs_2m1_313_10[ctr_0 + 1]; - _data_pdfs_tmp_20_314_10[ctr_0] = forceTerm_14 + xi_189 + xi_193 + - xi_194 + - _data_pdfs_2m1_314_10[ctr_0 - 1]; - _data_pdfs_tmp_20_315_10[ctr_0] = forceTerm_15 + xi_182 + xi_184 + - xi_196 + - _data_pdfs_21_315_1m1[ctr_0]; - _data_pdfs_tmp_20_316_10[ctr_0] = forceTerm_16 + xi_171 + xi_183 + - xi_196 + - _data_pdfs_21_316_11[ctr_0]; - _data_pdfs_tmp_20_317_10[ctr_0] = forceTerm_17 + xi_192 + xi_193 + - xi_197 + - _data_pdfs_21_317_10[ctr_0 + 1]; - _data_pdfs_tmp_20_318_10[ctr_0] = forceTerm_18 + xi_188 + xi_194 + - xi_197 + - _data_pdfs_21_318_10[ctr_0 - 1]; - } + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + { + for (int64_t ctr_0 = 1; ctr_0 < (int64_t)((_size_force_0 - 2) / (4)) * (4) + 1; ctr_0 += 4) + { + const __m256d xi_0 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1]),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_1 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_2 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_1),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])); + const __m256d xi_3 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_315_1m1[ctr_0]),_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_4 = _mm256_add_pd(xi_3,_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0])); + const __m256d xi_5 = _mm256_add_pd(xi_4,_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_6 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0]),_mm256_loadu_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_7 = _mm256_add_pd(xi_6,_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_8 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_9 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0]),_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_10 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]),_mm256_loadu_pd(& _data_pdfs_21_36_10[ctr_0])); + const __m256d xi_12 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + const __m256d xi_13 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])); + const __m256d xi_14 = _mm256_add_pd(xi_12,xi_13); + const __m256d xi_15 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_16 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_17 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_18 = _mm256_add_pd(xi_16,xi_17); + const __m256d xi_19 = _mm256_add_pd(xi_15,xi_18); + const __m256d xi_20 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_21 = _mm256_add_pd(xi_20,_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_22 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_23 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0])); + const __m256d xi_24 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_25 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_23),xi_24); + const __m256d xi_26 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_27 = _mm256_add_pd(xi_12,xi_26); + const __m256d xi_28 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_315_1m1[ctr_0])); + const __m256d xi_29 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_21_36_10[ctr_0])); + const __m256d xi_30 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_24,xi_28),xi_29),_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_31 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_32 = _mm256_add_pd(_mm256_add_pd(xi_31,xi_8),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_33 = _mm256_add_pd(_mm256_add_pd(xi_16,xi_21),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + const __m256d xi_34 = _mm256_add_pd(xi_26,_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + const __m256d xi_35 = _mm256_add_pd(xi_15,_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_36 = _mm256_add_pd(xi_34,xi_35); + const __m256d xi_37 = _mm256_add_pd(_mm256_add_pd(xi_9,_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])),_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_38 = _mm256_add_pd(xi_28,_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_39 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_38),_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_55 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_loadu_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_63 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_loadu_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_67 = _mm256_mul_pd(_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667),_mm256_loadu_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_70 = _mm256_mul_pd(_mm256_set_pd(0.5,0.5,0.5,0.5),_mm256_loadu_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_74 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_loadu_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_78 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_loadu_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_88 = _mm256_mul_pd(_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333),_mm256_loadu_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_99 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_30_10[ctr_0])); + const __m256d xi_100 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_loadu_pd(& _data_pdfs_21_36_10[ctr_0])),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_loadu_pd(& _data_pdfs_2m1_35_10[ctr_0]))),xi_99); + const __m256d xi_101 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_loadu_pd(& _data_pdfs_21_315_1m1[ctr_0])),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-3.0,-3.0,-3.0,-3.0),_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(3.0,3.0,3.0,3.0),_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_100),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_102 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_315_1m1[ctr_0])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0]))); + const __m256d xi_103 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))),xi_102); + const __m256d xi_104 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-5.0,-5.0,-5.0,-5.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_100),xi_103),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_107 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + const __m256d xi_108 = _mm256_add_pd(xi_107,xi_22); + const __m256d xi_109 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + const __m256d xi_112 = _mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_113 = _mm256_add_pd(_mm256_add_pd(xi_112,xi_19),xi_27); + const __m256d xi_115 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_116 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + const __m256d xi_117 = _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))); + const __m256d xi_118 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1]))),_mm256_mul_pd(_mm256_set_pd(-7.0,-7.0,-7.0,-7.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-4.0,-4.0,-4.0,-4.0),_mm256_loadu_pd(& _data_pdfs_21_36_10[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(-4.0,-4.0,-4.0,-4.0),_mm256_loadu_pd(& _data_pdfs_2m1_35_10[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0]))),_mm256_mul_pd(_mm256_set_pd(5.0,5.0,5.0,5.0),_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0]))),xi_103),xi_115),xi_116),xi_117),xi_99),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_119 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_107,xi_23),xi_38),_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0])),_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0])); + const __m256d xi_121 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_122 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + const __m256d xi_123 = _mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_124 = _mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])),_mm256_mul_pd(_mm256_set_pd(2.0,2.0,2.0,2.0),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))); + const __m256d xi_125 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_122,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_123),xi_124),xi_25),xi_4); + const __m256d xi_127 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_128 = _mm256_mul_pd(xi_127,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_133 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.0198412698412698,-0.0198412698412698,-0.0198412698412698,-0.0198412698412698)); + const __m256d xi_144 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_112,xi_13),xi_34),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + const __m256d xi_145 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_146 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_123,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_122),xi_124),xi_14),xi_35),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_147 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_149 = _mm256_mul_pd(xi_147,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_150 = _mm256_add_pd(_mm256_loadu_pd(& _data_pdfs_21_315_1m1[ctr_0]),_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d xi_151 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_108,xi_150),xi_29),_mm256_loadu_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_152 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_153 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_115,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_116,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_117),xi_30),xi_6); + const __m256d xi_154 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_155 = _mm256_mul_pd(xi_154,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_162 = _mm256_mul_pd(xi_118,_mm256_set_pd(0.0138888888888889,0.0138888888888889,0.0138888888888889,0.0138888888888889)); + const __m256d xi_177 = _mm256_mul_pd(xi_104,_mm256_set_pd(-0.00714285714285714,-0.00714285714285714,-0.00714285714285714,-0.00714285714285714)); + const __m256d xi_179 = _mm256_mul_pd(xi_101,_mm256_set_pd(0.025,0.025,0.025,0.025)); + const __m256d xi_182 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_184 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_185 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.00396825396825397,-0.00396825396825397,-0.00396825396825397,-0.00396825396825397)); + const __m256d xi_189 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_190 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_196 = _mm256_mul_pd(xi_104,_mm256_set_pd(0.0178571428571429,0.0178571428571429,0.0178571428571429,0.0178571428571429)); + const __m256d xi_199 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_200 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d vel0Term = xi_2; + const __m256d vel1Term = xi_5; + const __m256d vel2Term = xi_7; + const __m256d rho = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel0Term,vel1Term),vel2Term),xi_10),xi_8),xi_9),_mm256_loadu_pd(& _data_pdfs_20_30_10[ctr_0])); + const __m256d xi_11 = _mm256_div_pd(_mm256_set_pd(1.0,1.0,1.0,1.0),rho); + const __m256d u_0 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(vel0Term,xi_14),xi_19)); + const __m256d xi_40 = _mm256_mul_pd(u_0,_mm256_loadu_pd(& _data_force_20_30_10[ctr_0])); + const __m256d xi_41 = _mm256_mul_pd(xi_40,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_47 = _mm256_mul_pd(xi_41,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_105 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_0,u_0))); + const __m256d xi_141 = _mm256_mul_pd(rho,u_0); + const __m256d xi_142 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel0Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_141),xi_32); + const __m256d xi_143 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_167 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d u_1 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel1Term,xi_17),xi_21),xi_25)); + const __m256d xi_42 = _mm256_mul_pd(u_1,_mm256_loadu_pd(& _data_force_20_31_10[ctr_0])); + const __m256d xi_43 = _mm256_mul_pd(xi_42,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_48 = _mm256_mul_pd(xi_43,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_69 = _mm256_mul_pd(u_1,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_72 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0,xi_70),_mm256_mul_pd(xi_69,_mm256_loadu_pd(& _data_force_20_30_10[ctr_0]))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_73 = _mm256_mul_pd(xi_72,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_110 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_1,u_1))); + const __m256d xi_111 = _mm256_add_pd(_mm256_add_pd(xi_109,xi_110),xi_20); + const __m256d xi_129 = _mm256_mul_pd(rho,u_1); + const __m256d xi_130 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel1Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_109),xi_129),xi_37),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + const __m256d xi_132 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_165 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_166 = _mm256_add_pd(xi_127,xi_165); + const __m256d xi_175 = _mm256_mul_pd(xi_165,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_176 = _mm256_add_pd(xi_128,xi_175); + const __m256d xi_191 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_189,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_165),xi_190); + const __m256d xi_192 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_190,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_175),xi_189); + const __m256d u_2 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term,xi_27),xi_30),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))); + const __m256d xi_44 = _mm256_mul_pd(u_2,_mm256_loadu_pd(& _data_force_20_32_10[ctr_0])); + const __m256d xi_45 = _mm256_mul_pd(xi_44,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_46 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_41,xi_43),xi_45),_mm256_set_pd(-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0)); + const __m256d xi_49 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_44,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_48); + const __m256d xi_52 = _mm256_mul_pd(xi_45,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_53 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_42,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_52); + const __m256d xi_54 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_40,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_52); + const __m256d xi_57 = _mm256_mul_pd(xi_49,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_58 = _mm256_mul_pd(xi_57,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_59 = _mm256_mul_pd(xi_54,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_60 = _mm256_mul_pd(xi_59,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_62 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_60); + const __m256d xi_64 = _mm256_mul_pd(xi_53,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_65 = _mm256_mul_pd(xi_64,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_66 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_65); + const __m256d xi_68 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_60),xi_65); + const __m256d xi_75 = _mm256_add_pd(_mm256_mul_pd(xi_74,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_59); + const __m256d xi_77 = _mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_79 = _mm256_mul_pd(xi_46,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_80 = _mm256_add_pd(xi_64,xi_79); + const __m256d xi_81 = _mm256_add_pd(xi_78,xi_80); + const __m256d xi_82 = _mm256_add_pd(xi_77,xi_81); + const __m256d xi_83 = _mm256_add_pd(xi_59,xi_74); + const __m256d xi_84 = _mm256_add_pd(_mm256_mul_pd(xi_78,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_80); + const __m256d xi_85 = _mm256_add_pd(xi_77,xi_84); + const __m256d xi_86 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2,xi_70),_mm256_mul_pd(xi_69,_mm256_loadu_pd(& _data_force_20_32_10[ctr_0]))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_87 = _mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_89 = _mm256_add_pd(xi_57,xi_88); + const __m256d xi_90 = _mm256_add_pd(xi_87,xi_89); + const __m256d xi_91 = _mm256_mul_pd(xi_86,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_92 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0,_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_loadu_pd(& _data_force_20_32_10[ctr_0])),_mm256_mul_pd(_mm256_mul_pd(u_2,_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_loadu_pd(& _data_force_20_30_10[ctr_0]))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_93 = _mm256_mul_pd(xi_92,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_94 = _mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_95 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_89),xi_94); + const __m256d xi_96 = _mm256_add_pd(_mm256_mul_pd(xi_88,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_57); + const __m256d xi_97 = _mm256_add_pd(xi_87,xi_96); + const __m256d xi_98 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_94),xi_96); + const __m256d xi_106 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_2,u_2))); + const __m256d xi_114 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_105,xi_106),xi_108),xi_111),xi_113),xi_24),xi_28),_mm256_loadu_pd(& _data_pdfs_20_30_10[ctr_0])),_mm256_set_pd(omega_bulk,omega_bulk,omega_bulk,omega_bulk)); + const __m256d xi_134 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_106,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_loadu_pd(& _data_pdfs_21_36_10[ctr_0])),_mm256_loadu_pd(& _data_pdfs_2m1_35_10[ctr_0])); + const __m256d xi_135 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_set_pd(-1.0,-1.0,-1.0,-1.0),_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0])),xi_1),xi_111),xi_134),xi_18),xi_23),xi_31),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_136 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_137 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_105,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_110,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]))),_mm256_mul_pd(_mm256_set_pd(-2.0,-2.0,-2.0,-2.0),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))),xi_102),xi_109),xi_113),xi_134),xi_20),_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0])),_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0])),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_138 = _mm256_mul_pd(xi_137,_mm256_set_pd(-0.0416666666666667,-0.0416666666666667,-0.0416666666666667,-0.0416666666666667)); + const __m256d xi_139 = _mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-0.05,-0.05,-0.05,-0.05)),xi_138); + const __m256d xi_140 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(0.0142857142857143,0.0142857142857143,0.0142857142857143,0.0142857142857143)),xi_133),xi_136),xi_139); + const __m256d xi_148 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0357142857142857,-0.0357142857142857,-0.0357142857142857,-0.0357142857142857)),_mm256_mul_pd(xi_137,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))),xi_133); + const __m256d xi_156 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(rho,u_2),_mm256_mul_pd(vel2Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_10),xi_107),xi_112),xi_150),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + const __m256d xi_157 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_158 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0214285714285714,-0.0214285714285714,-0.0214285714285714,-0.0214285714285714)),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0158730158730159,0.0158730158730159,0.0158730158730159,0.0158730158730159))),_mm256_mul_pd(xi_136,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_139); + const __m256d xi_161 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.0625,0.0625,0.0625,0.0625)); + const __m256d xi_168 = _mm256_mul_pd(xi_114,_mm256_set_pd(0.0416666666666667,0.0416666666666667,0.0416666666666667,0.0416666666666667)); + const __m256d xi_169 = _mm256_add_pd(_mm256_mul_pd(xi_137,_mm256_set_pd(0.0208333333333333,0.0208333333333333,0.0208333333333333,0.0208333333333333)),xi_168); + const __m256d xi_170 = _mm256_add_pd(_mm256_mul_pd(xi_167,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_169); + const __m256d xi_171 = _mm256_add_pd(xi_149,xi_170); + const __m256d xi_173 = _mm256_add_pd(xi_167,xi_169); + const __m256d xi_174 = _mm256_add_pd(xi_147,xi_173); + const __m256d xi_186 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_187 = _mm256_add_pd(xi_185,xi_186); + const __m256d xi_188 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_182,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_184),xi_187); + const __m256d xi_195 = _mm256_mul_pd(xi_161,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_198 = _mm256_add_pd(xi_154,xi_187); + const __m256d xi_201 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_199,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_170),xi_200); + const __m256d xi_203 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_200,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_173),xi_199); + const __m256d xi_204 = _mm256_add_pd(_mm256_mul_pd(xi_186,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_185); + const __m256d xi_205 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_184,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_182),xi_204); + const __m256d xi_206 = _mm256_add_pd(xi_155,xi_204); + const __m256d p_0 = _mm256_add_pd(xi_2,xi_32); + const __m256d p_1 = xi_33; + const __m256d xi_160 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_1,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_0,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_163 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_160,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_161),xi_162); + const __m256d xi_172 = _mm256_add_pd(_mm256_add_pd(xi_160,xi_161),xi_162); + const __m256d p_2 = xi_36; + const __m256d xi_194 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_2,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_141)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_197 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_194,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_195),xi_196); + const __m256d xi_202 = _mm256_add_pd(_mm256_add_pd(xi_194,xi_195),xi_196); + const __m256d p_3 = xi_33; + const __m256d p_4 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_37),xi_5); + const __m256d p_5 = xi_39; + const __m256d xi_178 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_5,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_180 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_138,xi_168),xi_177),xi_178),xi_179); + const __m256d xi_193 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_178,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_138),xi_168),xi_177),xi_179); + const __m256d p_6 = xi_36; + const __m256d p_7 = xi_39; + const __m256d p_8 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_1,xi_10),xi_3),xi_7),_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0])); + const __m256d forceTerm_0 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_46,_mm256_set_pd(-1.5,-1.5,-1.5,-1.5)),_mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))); + const __m256d forceTerm_1 = _mm256_add_pd(xi_55,xi_62); + const __m256d forceTerm_2 = _mm256_add_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_62); + const __m256d forceTerm_3 = _mm256_add_pd(_mm256_mul_pd(xi_63,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_66); + const __m256d forceTerm_4 = _mm256_add_pd(xi_63,xi_66); + const __m256d forceTerm_5 = _mm256_add_pd(xi_67,xi_68); + const __m256d forceTerm_6 = _mm256_add_pd(_mm256_mul_pd(xi_67,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_68); + const __m256d forceTerm_7 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_75),xi_82); + const __m256d forceTerm_8 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_82),xi_83); + const __m256d forceTerm_9 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_75),xi_85); + const __m256d forceTerm_10 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_83),xi_85); + const __m256d forceTerm_11 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_86),xi_90); + const __m256d forceTerm_12 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_90),xi_91); + const __m256d forceTerm_13 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_93),xi_95); + const __m256d forceTerm_14 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_92),xi_95); + const __m256d forceTerm_15 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_91),xi_97); + const __m256d forceTerm_16 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_86),xi_97); + const __m256d forceTerm_17 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_92),xi_98); + const __m256d forceTerm_18 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_93),xi_98); + _mm256_storeu_pd(&_data_pdfs_tmp_20_30_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(0.1,0.1,0.1,0.1)),_mm256_mul_pd(xi_104,_mm256_set_pd(0.0428571428571429,0.0428571428571429,0.0428571428571429,0.0428571428571429))),_mm256_mul_pd(xi_114,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0238095238095238,0.0238095238095238,0.0238095238095238,0.0238095238095238))),forceTerm_0),_mm256_loadu_pd(& _data_pdfs_20_30_10[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_31_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_121,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_1),xi_128),xi_132),xi_140),_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_32_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_132,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_2),xi_121),xi_127),xi_140),_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_33_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_143,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_3),xi_145),xi_147),xi_148),_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_34_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_145,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_4),xi_143),xi_148),xi_149),_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_35_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_152,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_5),xi_155),xi_157),xi_158),_mm256_loadu_pd(& _data_pdfs_2m1_35_10[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_36_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_157,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_6),xi_152),xi_154),xi_158),_mm256_loadu_pd(& _data_pdfs_21_36_10[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_37_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_7,xi_163),xi_166),xi_171),_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_38_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_8,xi_166),xi_172),xi_174),_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_39_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_9,xi_171),xi_172),xi_176),_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_310_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_10,xi_163),xi_174),xi_176),_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_311_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_11,xi_180),xi_188),xi_191),_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_312_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_12,xi_188),xi_192),xi_193),_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_313_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_13,xi_197),xi_198),xi_201),_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_314_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_14,xi_198),xi_202),xi_203),_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_315_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_15,xi_191),xi_193),xi_205),_mm256_loadu_pd(& _data_pdfs_21_315_1m1[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_316_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_16,xi_180),xi_192),xi_205),_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_317_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_17,xi_201),xi_202),xi_206),_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1]))); + _mm256_storeu_pd(&_data_pdfs_tmp_20_318_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_18,xi_197),xi_203),xi_206),_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1]))); + } + for (int64_t ctr_0 = (int64_t)((_size_force_0 - 2) / (4)) * (4) + 1; ctr_0 < _size_force_0 - 1; ctr_0 += 1) + { + const double xi_0 = _data_pdfs_20_310_11[ctr_0 - 1] + _data_pdfs_20_38_1m1[ctr_0 - 1]; + const double xi_1 = _data_pdfs_21_318_10[ctr_0 - 1] + _data_pdfs_2m1_314_10[ctr_0 - 1]; + const double xi_2 = xi_0 + xi_1 + _data_pdfs_20_34_10[ctr_0 - 1]; + const double xi_3 = _data_pdfs_21_315_1m1[ctr_0] + _data_pdfs_2m1_311_1m1[ctr_0]; + const double xi_4 = xi_3 + _data_pdfs_20_31_1m1[ctr_0]; + const double xi_5 = xi_4 + _data_pdfs_20_37_1m1[ctr_0 + 1]; + const double xi_6 = _data_pdfs_2m1_312_11[ctr_0] + _data_pdfs_2m1_35_10[ctr_0]; + const double xi_7 = xi_6 + _data_pdfs_2m1_313_10[ctr_0 + 1]; + const double xi_8 = _data_pdfs_20_33_10[ctr_0 + 1] + _data_pdfs_20_39_11[ctr_0 + 1]; + const double xi_9 = _data_pdfs_20_32_11[ctr_0] + _data_pdfs_21_316_11[ctr_0]; + const double xi_10 = _data_pdfs_21_317_10[ctr_0 + 1] + _data_pdfs_21_36_10[ctr_0]; + const double xi_12 = -_data_pdfs_21_317_10[ctr_0 + 1]; + const double xi_13 = -_data_pdfs_20_33_10[ctr_0 + 1]; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -_data_pdfs_2m1_313_10[ctr_0 + 1]; + const double xi_16 = -_data_pdfs_20_37_1m1[ctr_0 + 1]; + const double xi_17 = -_data_pdfs_20_39_11[ctr_0 + 1]; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -_data_pdfs_20_310_11[ctr_0 - 1]; + const double xi_21 = xi_20 + _data_pdfs_20_38_1m1[ctr_0 - 1]; + const double xi_22 = -_data_pdfs_2m1_312_11[ctr_0]; + const double xi_23 = -_data_pdfs_20_32_11[ctr_0]; + const double xi_24 = -_data_pdfs_21_316_11[ctr_0]; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -_data_pdfs_21_318_10[ctr_0 - 1]; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -_data_pdfs_21_315_1m1[ctr_0]; + const double xi_29 = -_data_pdfs_21_36_10[ctr_0]; + const double xi_30 = xi_24 + xi_28 + xi_29 + _data_pdfs_2m1_311_1m1[ctr_0]; + const double xi_31 = _data_pdfs_21_317_10[ctr_0 + 1] + _data_pdfs_2m1_313_10[ctr_0 + 1]; + const double xi_32 = xi_31 + xi_8 + _data_pdfs_20_37_1m1[ctr_0 + 1]; + const double xi_33 = xi_16 + xi_21 + _data_pdfs_20_39_11[ctr_0 + 1]; + const double xi_34 = xi_26 + _data_pdfs_21_317_10[ctr_0 + 1]; + const double xi_35 = xi_15 + _data_pdfs_2m1_314_10[ctr_0 - 1]; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_9 + _data_pdfs_20_39_11[ctr_0 + 1] + _data_pdfs_2m1_312_11[ctr_0]; + const double xi_38 = xi_28 + _data_pdfs_21_316_11[ctr_0]; + const double xi_39 = xi_22 + xi_38 + _data_pdfs_2m1_311_1m1[ctr_0]; + const double xi_55 = 0.166666666666667*_data_force_20_31_10[ctr_0]; + const double xi_63 = 0.166666666666667*_data_force_20_30_10[ctr_0]; + const double xi_67 = 0.166666666666667*_data_force_20_32_10[ctr_0]; + const double xi_70 = 0.5*_data_force_20_31_10[ctr_0]; + const double xi_74 = 0.0833333333333333*_data_force_20_30_10[ctr_0]; + const double xi_78 = 0.0833333333333333*_data_force_20_31_10[ctr_0]; + const double xi_88 = 0.0833333333333333*_data_force_20_32_10[ctr_0]; + const double xi_99 = -_data_pdfs_20_30_10[ctr_0]; + const double xi_100 = xi_99 + 3.0*_data_pdfs_21_36_10[ctr_0] + 3.0*_data_pdfs_2m1_35_10[ctr_0]; + const double xi_101 = omega_even*(xi_100 - 3.0*_data_pdfs_21_315_1m1[ctr_0] - 3.0*_data_pdfs_21_316_11[ctr_0] - 3.0*_data_pdfs_2m1_311_1m1[ctr_0] - 3.0*_data_pdfs_2m1_312_11[ctr_0] + 3.0*_data_pdfs_20_31_1m1[ctr_0] + 3.0*_data_pdfs_20_32_11[ctr_0]); + const double xi_102 = 2.0*_data_pdfs_21_315_1m1[ctr_0] + 2.0*_data_pdfs_21_316_11[ctr_0] + 2.0*_data_pdfs_2m1_311_1m1[ctr_0] + 2.0*_data_pdfs_2m1_312_11[ctr_0]; + const double xi_103 = xi_102 + 5.0*_data_pdfs_20_33_10[ctr_0 + 1] + 5.0*_data_pdfs_20_34_10[ctr_0 - 1]; + const double xi_104 = omega_even*(xi_100 + xi_103 - 2.0*_data_pdfs_20_31_1m1[ctr_0] - 2.0*_data_pdfs_20_32_11[ctr_0] - 5.0*_data_pdfs_21_317_10[ctr_0 + 1] - 5.0*_data_pdfs_21_318_10[ctr_0 - 1] - 5.0*_data_pdfs_2m1_313_10[ctr_0 + 1] - 5.0*_data_pdfs_2m1_314_10[ctr_0 - 1]); + const double xi_107 = -_data_pdfs_2m1_311_1m1[ctr_0]; + const double xi_108 = xi_107 + xi_22; + const double xi_109 = -_data_pdfs_20_38_1m1[ctr_0 - 1]; + const double xi_112 = -_data_pdfs_2m1_314_10[ctr_0 - 1]; + const double xi_113 = xi_112 + xi_19 + xi_27; + const double xi_115 = 2.0*_data_pdfs_2m1_313_10[ctr_0 + 1]; + const double xi_116 = 2.0*_data_pdfs_2m1_314_10[ctr_0 - 1]; + const double xi_117 = 2.0*_data_pdfs_21_317_10[ctr_0 + 1] + 2.0*_data_pdfs_21_318_10[ctr_0 - 1]; + const double xi_118 = omega_even*(xi_103 + xi_115 + xi_116 + xi_117 + xi_99 - 4.0*_data_pdfs_21_36_10[ctr_0] - 4.0*_data_pdfs_2m1_35_10[ctr_0] - 7.0*_data_pdfs_20_310_11[ctr_0 - 1] - 7.0*_data_pdfs_20_37_1m1[ctr_0 + 1] - 7.0*_data_pdfs_20_38_1m1[ctr_0 - 1] - 7.0*_data_pdfs_20_39_11[ctr_0 + 1] + 5.0*_data_pdfs_20_31_1m1[ctr_0] + 5.0*_data_pdfs_20_32_11[ctr_0]); + const double xi_119 = xi_107 + xi_23 + xi_38 + _data_pdfs_20_31_1m1[ctr_0] + _data_pdfs_2m1_312_11[ctr_0]; + const double xi_121 = xi_119*xi_120; + const double xi_122 = 2.0*_data_pdfs_20_37_1m1[ctr_0 + 1]; + const double xi_123 = 2.0*_data_pdfs_20_310_11[ctr_0 - 1]; + const double xi_124 = -2.0*_data_pdfs_20_38_1m1[ctr_0 - 1] + 2.0*_data_pdfs_20_39_11[ctr_0 + 1]; + const double xi_125 = -xi_122 + xi_123 + xi_124 + xi_25 + xi_4; + const double xi_127 = xi_125*xi_126; + const double xi_128 = -xi_127; + const double xi_133 = xi_118*-0.0198412698412698; + const double xi_144 = xi_112 + xi_13 + xi_34 + _data_pdfs_20_34_10[ctr_0 - 1] + _data_pdfs_2m1_313_10[ctr_0 + 1]; + const double xi_145 = xi_120*xi_144; + const double xi_146 = xi_122 - xi_123 + xi_124 + xi_14 + xi_35 + _data_pdfs_20_34_10[ctr_0 - 1] + _data_pdfs_21_318_10[ctr_0 - 1]; + const double xi_147 = xi_126*xi_146; + const double xi_149 = -xi_147; + const double xi_150 = _data_pdfs_21_315_1m1[ctr_0] + _data_pdfs_21_316_11[ctr_0]; + const double xi_151 = xi_108 + xi_150 + xi_29 + _data_pdfs_2m1_35_10[ctr_0]; + const double xi_152 = xi_120*xi_151; + const double xi_153 = -xi_115 - xi_116 + xi_117 + xi_30 + xi_6; + const double xi_154 = xi_126*xi_153; + const double xi_155 = -xi_154; + const double xi_162 = xi_118*0.0138888888888889; + const double xi_177 = xi_104*-0.00714285714285714; + const double xi_179 = xi_101*0.025; + const double xi_182 = xi_153*xi_181; + const double xi_184 = xi_151*xi_183; + const double xi_185 = xi_118*-0.00396825396825397; + const double xi_189 = xi_125*xi_181; + const double xi_190 = xi_119*xi_183; + const double xi_196 = xi_104*0.0178571428571429; + const double xi_199 = xi_144*xi_183; + const double xi_200 = xi_146*xi_181; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_8 + xi_9 + _data_pdfs_20_30_10[ctr_0]; + const double xi_11 = 1 / (rho); + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_40 = u_0*_data_force_20_30_10[ctr_0]; + const double xi_41 = xi_40*0.333333333333333; + const double xi_47 = -xi_41; + const double xi_105 = rho*(u_0*u_0); + const double xi_141 = rho*u_0; + const double xi_142 = -vel0Term + xi_141 + xi_32; + const double xi_143 = xi_131*xi_142; + const double xi_167 = xi_142*xi_164; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_42 = u_1*_data_force_20_31_10[ctr_0]; + const double xi_43 = xi_42*0.333333333333333; + const double xi_48 = -xi_43; + const double xi_69 = u_1*0.5; + const double xi_72 = xi_71*(u_0*xi_70 + xi_69*_data_force_20_30_10[ctr_0]); + const double xi_73 = -xi_72; + const double xi_110 = rho*(u_1*u_1); + const double xi_111 = xi_109 + xi_110 + xi_20; + const double xi_129 = rho*u_1; + const double xi_130 = -vel1Term + xi_109 + xi_129 + xi_37 + _data_pdfs_20_310_11[ctr_0 - 1]; + const double xi_132 = xi_130*xi_131; + const double xi_165 = xi_130*xi_164; + const double xi_166 = xi_127 + xi_165; + const double xi_175 = -xi_165; + const double xi_176 = xi_128 + xi_175; + const double xi_191 = xi_165 - xi_189 + xi_190; + const double xi_192 = xi_175 + xi_189 - xi_190; + const double u_2 = xi_11*(vel2Term + xi_27 + xi_30 + _data_pdfs_2m1_314_10[ctr_0 - 1]); + const double xi_44 = u_2*_data_force_20_32_10[ctr_0]; + const double xi_45 = xi_44*0.333333333333333; + const double xi_46 = (-omega_bulk + 2.0)*(xi_41 + xi_43 + xi_45); + const double xi_49 = xi_44*0.666666666666667 + xi_47 + xi_48; + const double xi_52 = -xi_45; + const double xi_53 = xi_42*0.666666666666667 + xi_47 + xi_52; + const double xi_54 = xi_40*0.666666666666667 + xi_48 + xi_52; + const double xi_57 = xi_49*xi_56; + const double xi_58 = -xi_57; + const double xi_59 = xi_54*xi_56; + const double xi_60 = -xi_59; + const double xi_62 = xi_53*xi_61 + xi_58 + xi_60; + const double xi_64 = xi_53*xi_56; + const double xi_65 = -xi_64; + const double xi_66 = xi_54*xi_61 + xi_58 + xi_65; + const double xi_68 = xi_49*xi_61 + xi_60 + xi_65; + const double xi_75 = xi_59 - xi_74; + const double xi_77 = -xi_49*xi_76; + const double xi_79 = xi_46*0.125; + const double xi_80 = xi_64 + xi_79; + const double xi_81 = xi_78 + xi_80; + const double xi_82 = xi_77 + xi_81; + const double xi_83 = xi_59 + xi_74; + const double xi_84 = -xi_78 + xi_80; + const double xi_85 = xi_77 + xi_84; + const double xi_86 = xi_71*(u_2*xi_70 + xi_69*_data_force_20_32_10[ctr_0]); + const double xi_87 = -xi_54*xi_76; + const double xi_89 = xi_57 + xi_88; + const double xi_90 = xi_87 + xi_89; + const double xi_91 = -xi_86; + const double xi_92 = xi_71*(u_0*0.5*_data_force_20_32_10[ctr_0] + u_2*0.5*_data_force_20_30_10[ctr_0]); + const double xi_93 = -xi_92; + const double xi_94 = -xi_53*xi_76; + const double xi_95 = xi_79 + xi_89 + xi_94; + const double xi_96 = xi_57 - xi_88; + const double xi_97 = xi_87 + xi_96; + const double xi_98 = xi_79 + xi_94 + xi_96; + const double xi_106 = rho*(u_2*u_2); + const double xi_114 = omega_bulk*(xi_105 + xi_106 + xi_108 + xi_111 + xi_113 + xi_24 + xi_28 + _data_pdfs_20_30_10[ctr_0]); + const double xi_134 = -xi_106 + _data_pdfs_21_36_10[ctr_0] + _data_pdfs_2m1_35_10[ctr_0]; + const double xi_135 = omega_shear*(xi_1 + xi_111 + xi_134 + xi_18 + xi_23 + xi_31 - _data_pdfs_20_31_1m1[ctr_0]); + const double xi_136 = xi_135*0.125; + const double xi_137 = omega_shear*(xi_102 + xi_105*2.0 + xi_109 - xi_110 + xi_113 + xi_134 + xi_20 - 2.0*_data_pdfs_20_33_10[ctr_0 + 1] - 2.0*_data_pdfs_20_34_10[ctr_0 - 1] + _data_pdfs_20_31_1m1[ctr_0] + _data_pdfs_20_32_11[ctr_0]); + const double xi_138 = xi_137*-0.0416666666666667; + const double xi_139 = xi_101*-0.05 + xi_138; + const double xi_140 = xi_104*0.0142857142857143 + xi_133 + xi_136 + xi_139; + const double xi_148 = xi_104*-0.0357142857142857 + xi_133 + xi_137*0.0833333333333333; + const double xi_156 = rho*u_2 - vel2Term + xi_10 + xi_107 + xi_112 + xi_150 + _data_pdfs_21_318_10[ctr_0 - 1]; + const double xi_157 = xi_131*xi_156; + const double xi_158 = xi_104*-0.0214285714285714 + xi_118*0.0158730158730159 - xi_136 + xi_139; + const double xi_161 = xi_135*0.0625; + const double xi_168 = xi_114*0.0416666666666667; + const double xi_169 = xi_137*0.0208333333333333 + xi_168; + const double xi_170 = -xi_167 + xi_169; + const double xi_171 = xi_149 + xi_170; + const double xi_173 = xi_167 + xi_169; + const double xi_174 = xi_147 + xi_173; + const double xi_186 = xi_156*xi_164; + const double xi_187 = xi_185 + xi_186; + const double xi_188 = -xi_182 + xi_184 + xi_187; + const double xi_195 = -xi_161; + const double xi_198 = xi_154 + xi_187; + const double xi_201 = xi_170 - xi_199 + xi_200; + const double xi_203 = xi_173 + xi_199 - xi_200; + const double xi_204 = xi_185 - xi_186; + const double xi_205 = xi_182 - xi_184 + xi_204; + const double xi_206 = xi_155 + xi_204; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_160 = xi_159*(-p_1 + u_0*xi_129); + const double xi_163 = -xi_160 + xi_161 + xi_162; + const double xi_172 = xi_160 + xi_161 + xi_162; + const double p_2 = xi_36; + const double xi_194 = xi_159*(-p_2 + u_2*xi_141); + const double xi_197 = -xi_194 + xi_195 + xi_196; + const double xi_202 = xi_194 + xi_195 + xi_196; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_178 = xi_159*(-p_5 + u_2*xi_129); + const double xi_180 = xi_138 + xi_168 + xi_177 + xi_178 + xi_179; + const double xi_193 = xi_138 + xi_168 + xi_177 - xi_178 + xi_179; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_3 + xi_7 + _data_pdfs_21_316_11[ctr_0]; + const double forceTerm_0 = xi_46*-1.5 - xi_49*xi_51 - xi_51*xi_53 - xi_51*xi_54; + const double forceTerm_1 = xi_55 + xi_62; + const double forceTerm_2 = -xi_55 + xi_62; + const double forceTerm_3 = -xi_63 + xi_66; + const double forceTerm_4 = xi_63 + xi_66; + const double forceTerm_5 = xi_67 + xi_68; + const double forceTerm_6 = -xi_67 + xi_68; + const double forceTerm_7 = xi_73 + xi_75 + xi_82; + const double forceTerm_8 = xi_72 + xi_82 + xi_83; + const double forceTerm_9 = xi_72 + xi_75 + xi_85; + const double forceTerm_10 = xi_73 + xi_83 + xi_85; + const double forceTerm_11 = xi_81 + xi_86 + xi_90; + const double forceTerm_12 = xi_84 + xi_90 + xi_91; + const double forceTerm_13 = xi_75 + xi_93 + xi_95; + const double forceTerm_14 = xi_83 + xi_92 + xi_95; + const double forceTerm_15 = xi_81 + xi_91 + xi_97; + const double forceTerm_16 = xi_84 + xi_86 + xi_97; + const double forceTerm_17 = xi_75 + xi_92 + xi_98; + const double forceTerm_18 = xi_83 + xi_93 + xi_98; + _data_pdfs_tmp_20_30_10[ctr_0] = forceTerm_0 + xi_101*0.1 + xi_104*0.0428571428571429 + xi_114*-0.5 + xi_118*0.0238095238095238 + _data_pdfs_20_30_10[ctr_0]; + _data_pdfs_tmp_20_31_10[ctr_0] = forceTerm_1 - xi_121 + xi_128 + xi_132 + xi_140 + _data_pdfs_20_31_1m1[ctr_0]; + _data_pdfs_tmp_20_32_10[ctr_0] = forceTerm_2 + xi_121 + xi_127 - xi_132 + xi_140 + _data_pdfs_20_32_11[ctr_0]; + _data_pdfs_tmp_20_33_10[ctr_0] = forceTerm_3 - xi_143 + xi_145 + xi_147 + xi_148 + _data_pdfs_20_33_10[ctr_0 + 1]; + _data_pdfs_tmp_20_34_10[ctr_0] = forceTerm_4 + xi_143 - xi_145 + xi_148 + xi_149 + _data_pdfs_20_34_10[ctr_0 - 1]; + _data_pdfs_tmp_20_35_10[ctr_0] = forceTerm_5 - xi_152 + xi_155 + xi_157 + xi_158 + _data_pdfs_2m1_35_10[ctr_0]; + _data_pdfs_tmp_20_36_10[ctr_0] = forceTerm_6 + xi_152 + xi_154 - xi_157 + xi_158 + _data_pdfs_21_36_10[ctr_0]; + _data_pdfs_tmp_20_37_10[ctr_0] = forceTerm_7 + xi_163 + xi_166 + xi_171 + _data_pdfs_20_37_1m1[ctr_0 + 1]; + _data_pdfs_tmp_20_38_10[ctr_0] = forceTerm_8 + xi_166 + xi_172 + xi_174 + _data_pdfs_20_38_1m1[ctr_0 - 1]; + _data_pdfs_tmp_20_39_10[ctr_0] = forceTerm_9 + xi_171 + xi_172 + xi_176 + _data_pdfs_20_39_11[ctr_0 + 1]; + _data_pdfs_tmp_20_310_10[ctr_0] = forceTerm_10 + xi_163 + xi_174 + xi_176 + _data_pdfs_20_310_11[ctr_0 - 1]; + _data_pdfs_tmp_20_311_10[ctr_0] = forceTerm_11 + xi_180 + xi_188 + xi_191 + _data_pdfs_2m1_311_1m1[ctr_0]; + _data_pdfs_tmp_20_312_10[ctr_0] = forceTerm_12 + xi_188 + xi_192 + xi_193 + _data_pdfs_2m1_312_11[ctr_0]; + _data_pdfs_tmp_20_313_10[ctr_0] = forceTerm_13 + xi_197 + xi_198 + xi_201 + _data_pdfs_2m1_313_10[ctr_0 + 1]; + _data_pdfs_tmp_20_314_10[ctr_0] = forceTerm_14 + xi_198 + xi_202 + xi_203 + _data_pdfs_2m1_314_10[ctr_0 - 1]; + _data_pdfs_tmp_20_315_10[ctr_0] = forceTerm_15 + xi_191 + xi_193 + xi_205 + _data_pdfs_21_315_1m1[ctr_0]; + _data_pdfs_tmp_20_316_10[ctr_0] = forceTerm_16 + xi_180 + xi_192 + xi_205 + _data_pdfs_21_316_11[ctr_0]; + _data_pdfs_tmp_20_317_10[ctr_0] = forceTerm_17 + xi_201 + xi_202 + xi_206 + _data_pdfs_21_317_10[ctr_0 + 1]; + _data_pdfs_tmp_20_318_10[ctr_0] = forceTerm_18 + xi_197 + xi_203 + xi_206 + _data_pdfs_21_318_10[ctr_0 - 1]; + } + } } - } - } + } +} } -} // namespace internal_kernel_streamCollide namespace internal_kernel_collide { -static FUNC_PREFIX void -kernel_collide(double *RESTRICT const _data_force, double *RESTRICT _data_pdfs, - int64_t const _size_force_0, int64_t const _size_force_1, - int64_t const _size_force_2, int64_t const _stride_force_1, - int64_t const _stride_force_2, int64_t const _stride_force_3, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, double omega_bulk, - double omega_even, double omega_odd, double omega_shear) { - const double xi_35 = -omega_shear + 2.0; - const double xi_36 = xi_35 * 0.5; - const double xi_41 = xi_35 * 0.0833333333333333; - const double xi_46 = xi_35 * 0.166666666666667; - const double xi_56 = xi_35 * 0.25; - const double xi_61 = xi_35 * 0.0416666666666667; - const double xi_106 = omega_odd * 0.25; - const double xi_112 = omega_odd * 0.0833333333333333; - const double xi_149 = omega_shear * 0.25; - const double xi_172 = omega_odd * 0.0416666666666667; - const double xi_174 = omega_odd * 0.125; - const int64_t rr_0 = 0.0; - const double xi_118 = rr_0 * 0.166666666666667; - const double xi_154 = rr_0 * 0.0833333333333333; - for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_312 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_316 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_313 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_force_20_31 = - _data_force + _stride_force_2 * ctr_2 + _stride_force_3; - double *RESTRICT _data_pdfs_20_314 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_35 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_36 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_force_20_30 = _data_force + _stride_force_2 * ctr_2; - double *RESTRICT _data_pdfs_20_318 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_311 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_315 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_317 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_force_20_32 = - _data_force + _stride_force_2 * ctr_2 + 2 * _stride_force_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_312_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_312; - double *RESTRICT _data_pdfs_20_316_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_316; - double *RESTRICT _data_pdfs_20_31_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_20_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_313; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_20_37_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_37; - double *RESTRICT _data_force_20_31_10 = - _stride_force_1 * ctr_1 + _data_force_20_31; - double *RESTRICT _data_pdfs_20_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_314; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_20_39_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_20_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_35; - double *RESTRICT _data_pdfs_20_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_36; - double *RESTRICT _data_force_20_30_10 = - _stride_force_1 * ctr_1 + _data_force_20_30; - double *RESTRICT _data_pdfs_20_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_318; - double *RESTRICT _data_pdfs_20_38_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_20_311_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_311; - double *RESTRICT _data_pdfs_20_315_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_315; - double *RESTRICT _data_pdfs_20_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_317; - double *RESTRICT _data_pdfs_20_310_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_310; - double *RESTRICT _data_force_20_32_10 = - _stride_force_1 * ctr_1 + _data_force_20_32; - double *RESTRICT _data_pdfs_20_32_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_32; +static FUNC_PREFIX void kernel_collide(double * RESTRICT const _data_force, double * RESTRICT _data_pdfs, int64_t const _size_force_0, int64_t const _size_force_1, int64_t const _size_force_2, int64_t const _stride_force_1, int64_t const _stride_force_2, int64_t const _stride_force_3, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, double omega_bulk, double omega_even, double omega_odd, double omega_shear) +{ + const double xi_50 = -omega_shear + 2.0; + const double xi_51 = xi_50*0.5; + const double xi_56 = xi_50*0.0833333333333333; + const double xi_61 = xi_50*0.166666666666667; + const double xi_71 = xi_50*0.25; + const double xi_76 = xi_50*0.0416666666666667; + const double xi_120 = omega_odd*0.25; + const double xi_126 = omega_odd*0.0833333333333333; + const double xi_159 = omega_shear*0.25; + const double xi_181 = omega_odd*0.0416666666666667; + const double xi_183 = omega_odd*0.125; + const int64_t rr_0 = 0.0; + const double xi_131 = rr_0*0.166666666666667; + const double xi_164 = rr_0*0.0833333333333333; + for (int64_t ctr_2 = 0; ctr_2 < _size_force_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_force_20_30 = _data_force + _stride_force_2*ctr_2; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_force_20_31 = _data_force + _stride_force_2*ctr_2 + _stride_force_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_force_20_32 = _data_force + _stride_force_2*ctr_2 + 2*_stride_force_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_20_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_force_1; ctr_1 += 1) { - for (int64_t ctr_0 = 0; ctr_0 < (int64_t)((_size_force_0) / (4)) * (4); - ctr_0 += 4) { - const __m256d xi_198 = _mm256_loadu_pd(&_data_pdfs_20_312_10[ctr_0]); - const __m256d xi_199 = _mm256_loadu_pd(&_data_pdfs_20_316_10[ctr_0]); - const __m256d xi_200 = _mm256_loadu_pd(&_data_pdfs_20_31_10[ctr_0]); - const __m256d xi_201 = _mm256_loadu_pd(&_data_pdfs_20_313_10[ctr_0]); - const __m256d xi_202 = _mm256_loadu_pd(&_data_pdfs_20_30_10[ctr_0]); - const __m256d xi_203 = _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0]); - const __m256d xi_204 = _mm256_loadu_pd(&_data_pdfs_20_37_10[ctr_0]); - const __m256d xi_205 = _mm256_loadu_pd(&_data_force_20_31_10[ctr_0]); - const __m256d xi_206 = _mm256_loadu_pd(&_data_pdfs_20_314_10[ctr_0]); - const __m256d xi_207 = _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0]); - const __m256d xi_208 = _mm256_loadu_pd(&_data_pdfs_20_39_10[ctr_0]); - const __m256d xi_209 = _mm256_loadu_pd(&_data_pdfs_20_35_10[ctr_0]); - const __m256d xi_210 = _mm256_loadu_pd(&_data_pdfs_20_36_10[ctr_0]); - const __m256d xi_211 = _mm256_loadu_pd(&_data_force_20_30_10[ctr_0]); - const __m256d xi_212 = _mm256_loadu_pd(&_data_pdfs_20_318_10[ctr_0]); - const __m256d xi_213 = _mm256_loadu_pd(&_data_pdfs_20_38_10[ctr_0]); - const __m256d xi_214 = _mm256_loadu_pd(&_data_pdfs_20_311_10[ctr_0]); - const __m256d xi_215 = _mm256_loadu_pd(&_data_pdfs_20_315_10[ctr_0]); - const __m256d xi_216 = _mm256_loadu_pd(&_data_pdfs_20_317_10[ctr_0]); - const __m256d xi_217 = _mm256_loadu_pd(&_data_pdfs_20_310_10[ctr_0]); - const __m256d xi_218 = _mm256_loadu_pd(&_data_force_20_32_10[ctr_0]); - const __m256d xi_219 = _mm256_loadu_pd(&_data_pdfs_20_32_10[ctr_0]); - const __m256d xi_0 = _mm256_add_pd(xi_206, xi_212); - const __m256d xi_1 = _mm256_add_pd(xi_0, xi_203); - const __m256d xi_2 = - _mm256_add_pd(_mm256_add_pd(xi_200, xi_214), xi_215); - const __m256d xi_3 = _mm256_add_pd(xi_198, xi_209); - const __m256d xi_4 = _mm256_add_pd(xi_207, xi_208); - const __m256d xi_5 = _mm256_add_pd(xi_199, xi_219); - const __m256d xi_6 = _mm256_add_pd(xi_210, xi_216); - const __m256d xi_8 = - _mm256_mul_pd(xi_208, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_9 = _mm256_add_pd( - _mm256_mul_pd(xi_204, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_8); - const __m256d xi_10 = - _mm256_mul_pd(xi_216, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_11 = - _mm256_mul_pd(xi_201, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_12 = - _mm256_mul_pd(xi_207, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_13 = - _mm256_add_pd(_mm256_add_pd(xi_10, xi_11), xi_12); - const __m256d xi_14 = - _mm256_mul_pd(xi_219, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_15 = - _mm256_mul_pd(xi_217, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_16 = _mm256_add_pd(xi_14, xi_15); - const __m256d xi_17 = - _mm256_mul_pd(xi_199, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_18 = - _mm256_mul_pd(xi_198, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_19 = _mm256_add_pd(xi_17, xi_18); - const __m256d xi_20 = - _mm256_mul_pd(xi_212, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_21 = _mm256_add_pd(xi_10, xi_20); - const __m256d xi_22 = - _mm256_mul_pd(xi_215, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_23 = - _mm256_mul_pd(xi_210, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_24 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_17, xi_214), xi_22), xi_23); - const __m256d xi_40 = _mm256_mul_pd( - xi_205, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_48 = _mm256_mul_pd( - xi_211, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_52 = _mm256_mul_pd( - xi_218, _mm256_set_pd(0.166666666666667, 0.166666666666667, - 0.166666666666667, 0.166666666666667)); - const __m256d xi_55 = - _mm256_mul_pd(xi_205, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_59 = _mm256_mul_pd( - xi_211, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_63 = _mm256_mul_pd( - xi_205, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_73 = _mm256_mul_pd( - xi_218, _mm256_set_pd(0.0833333333333333, 0.0833333333333333, - 0.0833333333333333, 0.0833333333333333)); - const __m256d xi_84 = - _mm256_mul_pd(xi_202, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_85 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_209, _mm256_set_pd(3.0, 3.0, 3.0, 3.0)), - _mm256_mul_pd(xi_210, _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - xi_84); - const __m256d xi_86 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_198, - _mm256_set_pd(-3.0, -3.0, - -3.0, -3.0)), - _mm256_mul_pd(xi_199, - _mm256_set_pd(-3.0, -3.0, - -3.0, -3.0))), - _mm256_mul_pd( - xi_200, - _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - _mm256_mul_pd(xi_214, _mm256_set_pd(-3.0, -3.0, - -3.0, -3.0))), - _mm256_mul_pd(xi_215, - _mm256_set_pd(-3.0, -3.0, -3.0, -3.0))), - _mm256_mul_pd(xi_219, _mm256_set_pd(3.0, 3.0, 3.0, 3.0))), - xi_85), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_87 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_198, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)), - _mm256_mul_pd(xi_199, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))), - _mm256_mul_pd(xi_214, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))), - _mm256_mul_pd(xi_215, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_88 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_203, _mm256_set_pd(5.0, 5.0, 5.0, 5.0)), - _mm256_mul_pd(xi_207, _mm256_set_pd(5.0, 5.0, 5.0, 5.0))), - xi_87); - const __m256d xi_89 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_200, - _mm256_set_pd(-2.0, -2.0, -2.0, - -2.0)), - _mm256_mul_pd( - xi_201, - _mm256_set_pd(-5.0, -5.0, -5.0, - -5.0))), - _mm256_mul_pd(xi_206, - _mm256_set_pd(-5.0, -5.0, - -5.0, -5.0))), - _mm256_mul_pd( - xi_212, - _mm256_set_pd(-5.0, -5.0, -5.0, -5.0))), - _mm256_mul_pd(xi_216, _mm256_set_pd(-5.0, -5.0, - -5.0, -5.0))), - _mm256_mul_pd(xi_219, - _mm256_set_pd(-2.0, -2.0, -2.0, -2.0))), - xi_85), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_92 = - _mm256_mul_pd(xi_214, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_93 = _mm256_add_pd(xi_18, xi_92); - const __m256d xi_94 = - _mm256_mul_pd(xi_213, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_97 = - _mm256_mul_pd(xi_206, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_98 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_11, xi_15), xi_21), xi_97); - const __m256d xi_100 = - _mm256_mul_pd(xi_201, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_101 = - _mm256_mul_pd(xi_206, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_102 = _mm256_add_pd( - _mm256_mul_pd(xi_212, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)), - _mm256_mul_pd(xi_216, _mm256_set_pd(2.0, 2.0, 2.0, 2.0))); - const __m256d xi_103 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_200, - _mm256_set_pd( - 5.0, 5.0, - 5.0, - 5.0)), - _mm256_mul_pd( - xi_204, - _mm256_set_pd( - -7.0, - -7.0, - -7.0, - -7.0))), - _mm256_mul_pd( - xi_208, - _mm256_set_pd( - -7.0, -7.0, - -7.0, -7.0))), - _mm256_mul_pd( - xi_209, - _mm256_set_pd( - -4.0, -4.0, -4.0, - -4.0))), - _mm256_mul_pd( - xi_210, _mm256_set_pd( - -4.0, -4.0, - -4.0, -4.0))), - _mm256_mul_pd( - xi_213, - _mm256_set_pd(-7.0, -7.0, - -7.0, -7.0))), - _mm256_mul_pd( - xi_217, - _mm256_set_pd(-7.0, -7.0, -7.0, - -7.0))), - _mm256_mul_pd( - xi_219, - _mm256_set_pd(5.0, 5.0, 5.0, 5.0))), - xi_100), - xi_101), - xi_102), - xi_84), - xi_88), - _mm256_set_pd(omega_even, omega_even, omega_even, omega_even)); - const __m256d xi_104 = _mm256_add_pd(xi_198, xi_92); - const __m256d xi_105 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_104, xi_14), xi_199), - xi_200), - xi_22); - const __m256d xi_107 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_108 = - _mm256_mul_pd(xi_204, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_109 = - _mm256_mul_pd(xi_217, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)); - const __m256d xi_110 = _mm256_add_pd( - _mm256_mul_pd(xi_208, _mm256_set_pd(2.0, 2.0, 2.0, 2.0)), - _mm256_mul_pd(xi_213, _mm256_set_pd(-2.0, -2.0, -2.0, -2.0))); - const __m256d xi_111 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_108, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - xi_109), - xi_110), - xi_14), - xi_19), - xi_2); - const __m256d xi_113 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_114 = - _mm256_mul_pd(xi_113, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_116 = _mm256_add_pd(xi_217, xi_94); - const __m256d xi_120 = _mm256_add_pd(xi_201, xi_216); - const __m256d xi_124 = _mm256_mul_pd( - xi_103, _mm256_set_pd(-0.0198412698412698, -0.0198412698412698, - -0.0198412698412698, -0.0198412698412698)); - const __m256d xi_132 = _mm256_add_pd(xi_201, xi_97); - const __m256d xi_133 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_12, xi_132), xi_20), - xi_203), - xi_216); - const __m256d xi_134 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_135 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_109, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_1), - xi_108), - xi_110), - xi_13); - const __m256d xi_136 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_138 = - _mm256_mul_pd(xi_136, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_139 = _mm256_add_pd(xi_199, xi_215); - const __m256d xi_140 = _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_139, xi_209), xi_23), xi_93); - const __m256d xi_141 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_106, xi_106, xi_106, xi_106)); - const __m256d xi_144 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_100, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_mul_pd(xi_101, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_102), - xi_24), - xi_3); - const __m256d xi_145 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_112, xi_112, xi_112, xi_112)); - const __m256d xi_146 = - _mm256_mul_pd(xi_145, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_148 = xi_145; - const __m256d xi_152 = _mm256_mul_pd( - xi_103, _mm256_set_pd(0.0138888888888889, 0.0138888888888889, - 0.0138888888888889, 0.0138888888888889)); - const __m256d xi_168 = _mm256_mul_pd( - xi_89, _mm256_set_pd(-0.00714285714285714, -0.00714285714285714, - -0.00714285714285714, -0.00714285714285714)); - const __m256d xi_170 = - _mm256_mul_pd(xi_86, _mm256_set_pd(0.025, 0.025, 0.025, 0.025)); - const __m256d xi_173 = _mm256_mul_pd( - xi_144, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_175 = _mm256_mul_pd( - xi_140, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_176 = _mm256_mul_pd( - xi_103, - _mm256_set_pd(-0.00396825396825397, -0.00396825396825397, - -0.00396825396825397, -0.00396825396825397)); - const __m256d xi_180 = _mm256_mul_pd( - xi_111, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d xi_181 = _mm256_mul_pd( - xi_105, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_187 = _mm256_mul_pd( - xi_89, _mm256_set_pd(0.0178571428571429, 0.0178571428571429, - 0.0178571428571429, 0.0178571428571429)); - const __m256d xi_190 = _mm256_mul_pd( - xi_133, _mm256_set_pd(xi_174, xi_174, xi_174, xi_174)); - const __m256d xi_191 = _mm256_mul_pd( - xi_135, _mm256_set_pd(xi_172, xi_172, xi_172, xi_172)); - const __m256d vel0Term = - _mm256_add_pd(_mm256_add_pd(xi_1, xi_213), xi_217); - const __m256d vel1Term = _mm256_add_pd(xi_2, xi_204); - const __m256d vel2Term = _mm256_add_pd(xi_201, xi_3); - const __m256d rho = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel0Term, vel1Term), - vel2Term), - xi_202), - xi_4), - xi_5), - xi_6); - const __m256d xi_7 = - _mm256_div_pd(_mm256_set_pd(1.0, 1.0, 1.0, 1.0), rho); - const __m256d u_0 = _mm256_mul_pd( - xi_7, _mm256_add_pd(_mm256_add_pd(vel0Term, xi_13), xi_9)); - const __m256d xi_25 = _mm256_mul_pd(u_0, xi_211); - const __m256d xi_26 = _mm256_mul_pd( - xi_25, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_32 = - _mm256_mul_pd(xi_26, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_90 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_0, u_0))); - const __m256d xi_129 = _mm256_mul_pd(rho, u_0); - const __m256d xi_130 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(vel0Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_120), - xi_129), - xi_204), - xi_4); - const __m256d xi_131 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_158 = _mm256_mul_pd( - xi_130, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d u_1 = _mm256_mul_pd( - xi_7, - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel1Term, xi_16), xi_19), - xi_213), - xi_8)); - const __m256d xi_27 = _mm256_mul_pd(u_1, xi_205); - const __m256d xi_28 = _mm256_mul_pd( - xi_27, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_33 = - _mm256_mul_pd(xi_28, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_54 = - _mm256_mul_pd(u_1, _mm256_set_pd(0.5, 0.5, 0.5, 0.5)); - const __m256d xi_57 = - _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0, xi_55), - _mm256_mul_pd(xi_211, xi_54)), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_58 = - _mm256_mul_pd(xi_57, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_95 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_1, u_1))); - const __m256d xi_96 = - _mm256_add_pd(_mm256_add_pd(xi_9, xi_94), xi_95); - const __m256d xi_115 = _mm256_mul_pd(rho, u_1); - const __m256d xi_117 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - vel1Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_115), - xi_116), - xi_198), - xi_208), - xi_5); - const __m256d xi_119 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_150 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(u_0, xi_115), xi_116), - xi_204), - xi_8), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_155 = _mm256_mul_pd( - xi_117, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_156 = xi_155; - const __m256d xi_157 = _mm256_add_pd(xi_113, xi_156); - const __m256d xi_166 = - _mm256_mul_pd(xi_155, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_167 = _mm256_add_pd(xi_114, xi_166); - const __m256d xi_182 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_180, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_156), - xi_181); - const __m256d xi_183 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_181, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_166), - xi_180); - const __m256d u_2 = _mm256_mul_pd( - xi_7, _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(vel2Term, xi_206), xi_21), - xi_24)); - const __m256d xi_29 = _mm256_mul_pd(u_2, xi_218); - const __m256d xi_30 = _mm256_mul_pd( - xi_29, _mm256_set_pd(0.333333333333333, 0.333333333333333, - 0.333333333333333, 0.333333333333333)); - const __m256d xi_31 = _mm256_mul_pd( - _mm256_add_pd(_mm256_add_pd(xi_26, xi_28), xi_30), - _mm256_set_pd(-omega_bulk + 2.0, -omega_bulk + 2.0, - -omega_bulk + 2.0, -omega_bulk + 2.0)); - const __m256d xi_34 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_29, _mm256_set_pd(0.666666666666667, - 0.666666666666667, - 0.666666666666667, - 0.666666666666667)), - xi_32), - xi_33); - const __m256d xi_37 = - _mm256_mul_pd(xi_30, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_38 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_27, _mm256_set_pd(0.666666666666667, - 0.666666666666667, - 0.666666666666667, - 0.666666666666667)), - xi_32), - xi_37); - const __m256d xi_39 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_25, _mm256_set_pd(0.666666666666667, - 0.666666666666667, - 0.666666666666667, - 0.666666666666667)), - xi_33), - xi_37); - const __m256d xi_42 = - _mm256_mul_pd(xi_34, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_43 = - _mm256_mul_pd(xi_42, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_44 = - _mm256_mul_pd(xi_39, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_45 = - _mm256_mul_pd(xi_44, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_47 = _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(xi_38, _mm256_set_pd(xi_46, xi_46, - xi_46, xi_46)), - xi_43), - xi_45); - const __m256d xi_49 = - _mm256_mul_pd(xi_38, _mm256_set_pd(xi_41, xi_41, xi_41, xi_41)); - const __m256d xi_50 = - _mm256_mul_pd(xi_49, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_51 = _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(xi_39, _mm256_set_pd(xi_46, xi_46, - xi_46, xi_46)), - xi_43), - xi_50); - const __m256d xi_53 = _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(xi_34, _mm256_set_pd(xi_46, xi_46, - xi_46, xi_46)), - xi_45), - xi_50); - const __m256d xi_60 = _mm256_add_pd( - _mm256_mul_pd(xi_59, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_44); - const __m256d xi_62 = _mm256_mul_pd( - _mm256_mul_pd(xi_34, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_64 = - _mm256_mul_pd(xi_31, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_65 = _mm256_add_pd(xi_49, xi_64); - const __m256d xi_66 = _mm256_add_pd(xi_63, xi_65); - const __m256d xi_67 = _mm256_add_pd(xi_62, xi_66); - const __m256d xi_68 = _mm256_add_pd(xi_44, xi_59); - const __m256d xi_69 = _mm256_add_pd( - _mm256_mul_pd(xi_63, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_65); - const __m256d xi_70 = _mm256_add_pd(xi_62, xi_69); - const __m256d xi_71 = - _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2, xi_55), - _mm256_mul_pd(xi_218, xi_54)), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_72 = _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_74 = _mm256_add_pd(xi_42, xi_73); - const __m256d xi_75 = _mm256_add_pd(xi_72, xi_74); - const __m256d xi_76 = - _mm256_mul_pd(xi_71, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_77 = _mm256_mul_pd( - _mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0, xi_218), - _mm256_set_pd(0.5, 0.5, 0.5, 0.5)), - _mm256_mul_pd(_mm256_mul_pd(u_2, xi_211), - _mm256_set_pd(0.5, 0.5, 0.5, 0.5))), - _mm256_set_pd(xi_56, xi_56, xi_56, xi_56)); - const __m256d xi_78 = - _mm256_mul_pd(xi_77, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_79 = _mm256_mul_pd( - _mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_61, xi_61, xi_61, xi_61)); - const __m256d xi_80 = - _mm256_add_pd(_mm256_add_pd(xi_64, xi_74), xi_79); - const __m256d xi_81 = _mm256_add_pd( - _mm256_mul_pd(xi_73, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_42); - const __m256d xi_82 = _mm256_add_pd(xi_72, xi_81); - const __m256d xi_83 = - _mm256_add_pd(_mm256_add_pd(xi_64, xi_79), xi_81); - const __m256d xi_91 = _mm256_mul_pd(rho, (_mm256_mul_pd(u_2, u_2))); - const __m256d xi_99 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_17, xi_202), - xi_22), - xi_90), - xi_91), - xi_93), - xi_96), - xi_98), - _mm256_set_pd(omega_bulk, omega_bulk, omega_bulk, omega_bulk)); - const __m256d xi_121 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_91, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_209), - xi_210); - const __m256d xi_122 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_200, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_0), - xi_120), - xi_121), - xi_16), - xi_96), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, - omega_shear)); - const __m256d xi_123 = - _mm256_mul_pd(xi_122, _mm256_set_pd(0.125, 0.125, 0.125, 0.125)); - const __m256d xi_125 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_203, - _mm256_set_pd( - -2.0, -2.0, -2.0, - -2.0)), - _mm256_mul_pd( - xi_207, - _mm256_set_pd( - -2.0, -2.0, -2.0, - -2.0))), - _mm256_mul_pd( - xi_90, - _mm256_set_pd(2.0, 2.0, - 2.0, 2.0))), - _mm256_mul_pd( - xi_95, - _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0))), - xi_121), - xi_200), - xi_219), - xi_87), - xi_9), - xi_94), - xi_98), - _mm256_set_pd(omega_shear, omega_shear, omega_shear, - omega_shear)); - const __m256d xi_126 = _mm256_mul_pd( - xi_125, _mm256_set_pd(-0.0416666666666667, -0.0416666666666667, - -0.0416666666666667, -0.0416666666666667)); - const __m256d xi_127 = _mm256_add_pd( - _mm256_mul_pd(xi_86, _mm256_set_pd(-0.05, -0.05, -0.05, -0.05)), - xi_126); - const __m256d xi_128 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_89, _mm256_set_pd(0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143, - 0.0142857142857143)), - xi_123), - xi_124), - xi_127); - const __m256d xi_137 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_125, _mm256_set_pd(0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333, - 0.0833333333333333)), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857, - -0.0357142857142857))), - xi_124); - const __m256d xi_142 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(rho, u_2), - _mm256_mul_pd( - vel2Term, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - xi_139), - xi_212), - xi_6), - xi_92), - xi_97); - const __m256d xi_143 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_118, xi_118, xi_118, xi_118)); - const __m256d xi_147 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_103, _mm256_set_pd(0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159, - 0.0158730158730159)), - _mm256_mul_pd(xi_123, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0))), - _mm256_mul_pd(xi_89, _mm256_set_pd(-0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714, - -0.0214285714285714))), - xi_127); - const __m256d xi_151 = _mm256_mul_pd( - xi_122, _mm256_set_pd(0.0625, 0.0625, 0.0625, 0.0625)); - const __m256d xi_153 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_150, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_151), - xi_152); - const __m256d xi_159 = _mm256_mul_pd( - xi_99, _mm256_set_pd(0.0416666666666667, 0.0416666666666667, - 0.0416666666666667, 0.0416666666666667)); - const __m256d xi_160 = _mm256_add_pd( - _mm256_mul_pd(xi_125, _mm256_set_pd(0.0208333333333333, - 0.0208333333333333, - 0.0208333333333333, - 0.0208333333333333)), - xi_159); - const __m256d xi_161 = _mm256_add_pd( - _mm256_mul_pd(xi_158, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_160); - const __m256d xi_162 = _mm256_add_pd(xi_138, xi_161); - const __m256d xi_163 = - _mm256_add_pd(_mm256_add_pd(xi_150, xi_151), xi_152); - const __m256d xi_164 = _mm256_add_pd(xi_158, xi_160); - const __m256d xi_165 = _mm256_add_pd(xi_136, xi_164); - const __m256d xi_169 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(u_2, xi_115), xi_104), xi_17), - xi_215), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_171 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(xi_126, xi_159), xi_168), xi_169), - xi_170); - const __m256d xi_177 = _mm256_mul_pd( - xi_142, _mm256_set_pd(xi_154, xi_154, xi_154, xi_154)); - const __m256d xi_178 = _mm256_add_pd(xi_176, xi_177); - const __m256d xi_179 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_173, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_175), - xi_178); - const __m256d xi_184 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_169, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_126), - xi_159), - xi_168), - xi_170); - const __m256d xi_185 = _mm256_mul_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_mul_pd(u_2, xi_129), xi_10), xi_132), - xi_212), - _mm256_set_pd(xi_149, xi_149, xi_149, xi_149)); - const __m256d xi_186 = - _mm256_mul_pd(xi_151, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)); - const __m256d xi_188 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_185, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_186), - xi_187); - const __m256d xi_189 = _mm256_add_pd(xi_148, xi_178); - const __m256d xi_192 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_190, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_161), - xi_191); - const __m256d xi_193 = - _mm256_add_pd(_mm256_add_pd(xi_185, xi_186), xi_187); - const __m256d xi_194 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_191, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_164), - xi_190); - const __m256d xi_195 = _mm256_add_pd( - _mm256_mul_pd(xi_177, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_176); - const __m256d xi_196 = _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_175, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_173), - xi_195); - const __m256d xi_197 = _mm256_add_pd(xi_146, xi_195); - const __m256d forceTerm_0 = _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd(xi_31, - _mm256_set_pd(-1.5, -1.5, -1.5, -1.5)), - _mm256_mul_pd( - _mm256_mul_pd(xi_34, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd(_mm256_mul_pd(xi_38, _mm256_set_pd(-1.0, -1.0, - -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))), - _mm256_mul_pd( - _mm256_mul_pd(xi_39, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - _mm256_set_pd(xi_36, xi_36, xi_36, xi_36))); - const __m256d forceTerm_1 = _mm256_add_pd(xi_40, xi_47); - const __m256d forceTerm_2 = _mm256_add_pd( - _mm256_mul_pd(xi_40, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_47); - const __m256d forceTerm_3 = _mm256_add_pd( - _mm256_mul_pd(xi_48, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_51); - const __m256d forceTerm_4 = _mm256_add_pd(xi_48, xi_51); - const __m256d forceTerm_5 = _mm256_add_pd(xi_52, xi_53); - const __m256d forceTerm_6 = _mm256_add_pd( - _mm256_mul_pd(xi_52, _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - xi_53); - const __m256d forceTerm_7 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_60), xi_67); - const __m256d forceTerm_8 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_67), xi_68); - const __m256d forceTerm_9 = - _mm256_add_pd(_mm256_add_pd(xi_57, xi_60), xi_70); - const __m256d forceTerm_10 = - _mm256_add_pd(_mm256_add_pd(xi_58, xi_68), xi_70); - const __m256d forceTerm_11 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_71), xi_75); - const __m256d forceTerm_12 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_75), xi_76); - const __m256d forceTerm_13 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_78), xi_80); - const __m256d forceTerm_14 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_77), xi_80); - const __m256d forceTerm_15 = - _mm256_add_pd(_mm256_add_pd(xi_66, xi_76), xi_82); - const __m256d forceTerm_16 = - _mm256_add_pd(_mm256_add_pd(xi_69, xi_71), xi_82); - const __m256d forceTerm_17 = - _mm256_add_pd(_mm256_add_pd(xi_60, xi_77), xi_83); - const __m256d forceTerm_18 = - _mm256_add_pd(_mm256_add_pd(xi_68, xi_78), xi_83); - _mm256_storeu_pd( - &_data_pdfs_20_30_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_103, - _mm256_set_pd(0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238, - 0.0238095238095238)), - _mm256_mul_pd( - xi_86, - _mm256_set_pd(0.1, 0.1, 0.1, 0.1))), - _mm256_mul_pd(xi_89, - _mm256_set_pd(0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429, - 0.0428571428571429))), - _mm256_mul_pd(xi_99, - _mm256_set_pd(-0.5, -0.5, -0.5, -0.5))), - forceTerm_0), - xi_202)); - _mm256_storeu_pd( - &_data_pdfs_20_31_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_107, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_1), - xi_114), - xi_119), - xi_128), - xi_200)); - _mm256_storeu_pd( - &_data_pdfs_20_32_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_119, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_2), - xi_107), - xi_113), - xi_128), - xi_219)); - _mm256_storeu_pd( - &_data_pdfs_20_33_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_131, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_3), - xi_134), - xi_136), - xi_137), - xi_207)); - _mm256_storeu_pd( - &_data_pdfs_20_34_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_134, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_4), - xi_131), - xi_137), - xi_138), - xi_203)); - _mm256_storeu_pd( - &_data_pdfs_20_35_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_141, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_5), - xi_143), - xi_146), - xi_147), - xi_209)); - _mm256_storeu_pd( - &_data_pdfs_20_36_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd( - _mm256_mul_pd( - xi_143, - _mm256_set_pd(-1.0, -1.0, -1.0, -1.0)), - forceTerm_6), - xi_141), - xi_147), - xi_148), - xi_210)); - _mm256_storeu_pd( - &_data_pdfs_20_37_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_7, xi_153), xi_157), - xi_162), - xi_204)); - _mm256_storeu_pd( - &_data_pdfs_20_38_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_8, xi_157), xi_163), - xi_165), - xi_213)); - _mm256_storeu_pd( - &_data_pdfs_20_39_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_9, xi_162), xi_163), - xi_167), - xi_208)); - _mm256_storeu_pd( - &_data_pdfs_20_310_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_10, xi_153), - xi_165), - xi_167), - xi_217)); - _mm256_storeu_pd( - &_data_pdfs_20_311_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_11, xi_171), - xi_179), - xi_182), - xi_214)); - _mm256_storeu_pd( - &_data_pdfs_20_312_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_12, xi_179), - xi_183), - xi_184), - xi_198)); - _mm256_storeu_pd( - &_data_pdfs_20_313_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_13, xi_188), - xi_189), - xi_192), - xi_201)); - _mm256_storeu_pd( - &_data_pdfs_20_314_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_14, xi_189), - xi_193), - xi_194), - xi_206)); - _mm256_storeu_pd( - &_data_pdfs_20_315_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_15, xi_182), - xi_184), - xi_196), - xi_215)); - _mm256_storeu_pd( - &_data_pdfs_20_316_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_16, xi_171), - xi_183), - xi_196), - xi_199)); - _mm256_storeu_pd( - &_data_pdfs_20_317_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_17, xi_192), - xi_193), - xi_197), - xi_216)); - _mm256_storeu_pd( - &_data_pdfs_20_318_10[ctr_0], - _mm256_add_pd( - _mm256_add_pd( - _mm256_add_pd(_mm256_add_pd(forceTerm_18, xi_188), - xi_194), - xi_197), - xi_212)); - } - for (int64_t ctr_0 = (int64_t)((_size_force_0) / (4)) * (4); - ctr_0 < _size_force_0; ctr_0 += 1) { - const double xi_198 = _data_pdfs_20_312_10[ctr_0]; - const double xi_199 = _data_pdfs_20_316_10[ctr_0]; - const double xi_200 = _data_pdfs_20_31_10[ctr_0]; - const double xi_201 = _data_pdfs_20_313_10[ctr_0]; - const double xi_202 = _data_pdfs_20_30_10[ctr_0]; - const double xi_203 = _data_pdfs_20_34_10[ctr_0]; - const double xi_204 = _data_pdfs_20_37_10[ctr_0]; - const double xi_205 = _data_force_20_31_10[ctr_0]; - const double xi_206 = _data_pdfs_20_314_10[ctr_0]; - const double xi_207 = _data_pdfs_20_33_10[ctr_0]; - const double xi_208 = _data_pdfs_20_39_10[ctr_0]; - const double xi_209 = _data_pdfs_20_35_10[ctr_0]; - const double xi_210 = _data_pdfs_20_36_10[ctr_0]; - const double xi_211 = _data_force_20_30_10[ctr_0]; - const double xi_212 = _data_pdfs_20_318_10[ctr_0]; - const double xi_213 = _data_pdfs_20_38_10[ctr_0]; - const double xi_214 = _data_pdfs_20_311_10[ctr_0]; - const double xi_215 = _data_pdfs_20_315_10[ctr_0]; - const double xi_216 = _data_pdfs_20_317_10[ctr_0]; - const double xi_217 = _data_pdfs_20_310_10[ctr_0]; - const double xi_218 = _data_force_20_32_10[ctr_0]; - const double xi_219 = _data_pdfs_20_32_10[ctr_0]; - const double xi_0 = xi_206 + xi_212; - const double xi_1 = xi_0 + xi_203; - const double xi_2 = xi_200 + xi_214 + xi_215; - const double xi_3 = xi_198 + xi_209; - const double xi_4 = xi_207 + xi_208; - const double xi_5 = xi_199 + xi_219; - const double xi_6 = xi_210 + xi_216; - const double xi_8 = -xi_208; - const double xi_9 = -xi_204 + xi_8; - const double xi_10 = -xi_216; - const double xi_11 = -xi_201; - const double xi_12 = -xi_207; - const double xi_13 = xi_10 + xi_11 + xi_12; - const double xi_14 = -xi_219; - const double xi_15 = -xi_217; - const double xi_16 = xi_14 + xi_15; - const double xi_17 = -xi_199; - const double xi_18 = -xi_198; - const double xi_19 = xi_17 + xi_18; - const double xi_20 = -xi_212; - const double xi_21 = xi_10 + xi_20; - const double xi_22 = -xi_215; - const double xi_23 = -xi_210; - const double xi_24 = xi_17 + xi_214 + xi_22 + xi_23; - const double xi_40 = xi_205 * 0.166666666666667; - const double xi_48 = xi_211 * 0.166666666666667; - const double xi_52 = xi_218 * 0.166666666666667; - const double xi_55 = xi_205 * 0.5; - const double xi_59 = xi_211 * 0.0833333333333333; - const double xi_63 = xi_205 * 0.0833333333333333; - const double xi_73 = xi_218 * 0.0833333333333333; - const double xi_84 = -xi_202; - const double xi_85 = xi_209 * 3.0 + xi_210 * 3.0 + xi_84; - const double xi_86 = - omega_even * - (xi_198 * -3.0 + xi_199 * -3.0 + xi_200 * 3.0 + xi_214 * -3.0 + - xi_215 * -3.0 + xi_219 * 3.0 + xi_85); - const double xi_87 = - xi_198 * 2.0 + xi_199 * 2.0 + xi_214 * 2.0 + xi_215 * 2.0; - const double xi_88 = xi_203 * 5.0 + xi_207 * 5.0 + xi_87; - const double xi_89 = - omega_even * - (xi_200 * -2.0 + xi_201 * -5.0 + xi_206 * -5.0 + xi_212 * -5.0 + - xi_216 * -5.0 + xi_219 * -2.0 + xi_85 + xi_88); - const double xi_92 = -xi_214; - const double xi_93 = xi_18 + xi_92; - const double xi_94 = -xi_213; - const double xi_97 = -xi_206; - const double xi_98 = xi_11 + xi_15 + xi_21 + xi_97; - const double xi_100 = xi_201 * 2.0; - const double xi_101 = xi_206 * 2.0; - const double xi_102 = xi_212 * 2.0 + xi_216 * 2.0; - const double xi_103 = - omega_even * - (xi_100 + xi_101 + xi_102 + xi_200 * 5.0 + xi_204 * -7.0 + - xi_208 * -7.0 + xi_209 * -4.0 + xi_210 * -4.0 + xi_213 * -7.0 + - xi_217 * -7.0 + xi_219 * 5.0 + xi_84 + xi_88); - const double xi_104 = xi_198 + xi_92; - const double xi_105 = xi_104 + xi_14 + xi_199 + xi_200 + xi_22; - const double xi_107 = xi_105 * xi_106; - const double xi_108 = xi_204 * 2.0; - const double xi_109 = xi_217 * 2.0; - const double xi_110 = xi_208 * 2.0 + xi_213 * -2.0; - const double xi_111 = - -xi_108 + xi_109 + xi_110 + xi_14 + xi_19 + xi_2; - const double xi_113 = xi_111 * xi_112; - const double xi_114 = -xi_113; - const double xi_116 = xi_217 + xi_94; - const double xi_120 = xi_201 + xi_216; - const double xi_124 = xi_103 * -0.0198412698412698; - const double xi_132 = xi_201 + xi_97; - const double xi_133 = xi_12 + xi_132 + xi_20 + xi_203 + xi_216; - const double xi_134 = xi_106 * xi_133; - const double xi_135 = xi_1 + xi_108 - xi_109 + xi_110 + xi_13; - const double xi_136 = xi_112 * xi_135; - const double xi_138 = -xi_136; - const double xi_139 = xi_199 + xi_215; - const double xi_140 = xi_139 + xi_209 + xi_23 + xi_93; - const double xi_141 = xi_106 * xi_140; - const double xi_144 = -xi_100 - xi_101 + xi_102 + xi_24 + xi_3; - const double xi_145 = xi_112 * xi_144; - const double xi_146 = -xi_145; - const double xi_148 = xi_145; - const double xi_152 = xi_103 * 0.0138888888888889; - const double xi_168 = xi_89 * -0.00714285714285714; - const double xi_170 = xi_86 * 0.025; - const double xi_173 = xi_144 * xi_172; - const double xi_175 = xi_140 * xi_174; - const double xi_176 = xi_103 * -0.00396825396825397; - const double xi_180 = xi_111 * xi_172; - const double xi_181 = xi_105 * xi_174; - const double xi_187 = xi_89 * 0.0178571428571429; - const double xi_190 = xi_133 * xi_174; - const double xi_191 = xi_135 * xi_172; - const double vel0Term = xi_1 + xi_213 + xi_217; - const double vel1Term = xi_2 + xi_204; - const double vel2Term = xi_201 + xi_3; - const double rho = - vel0Term + vel1Term + vel2Term + xi_202 + xi_4 + xi_5 + xi_6; - const double xi_7 = 1 / (rho); - const double u_0 = xi_7 * (vel0Term + xi_13 + xi_9); - const double xi_25 = u_0 * xi_211; - const double xi_26 = xi_25 * 0.333333333333333; - const double xi_32 = -xi_26; - const double xi_90 = rho * (u_0 * u_0); - const double xi_129 = rho * u_0; - const double xi_130 = -vel0Term + xi_120 + xi_129 + xi_204 + xi_4; - const double xi_131 = xi_118 * xi_130; - const double xi_158 = xi_130 * xi_154; - const double u_1 = xi_7 * (vel1Term + xi_16 + xi_19 + xi_213 + xi_8); - const double xi_27 = u_1 * xi_205; - const double xi_28 = xi_27 * 0.333333333333333; - const double xi_33 = -xi_28; - const double xi_54 = u_1 * 0.5; - const double xi_57 = xi_56 * (u_0 * xi_55 + xi_211 * xi_54); - const double xi_58 = -xi_57; - const double xi_95 = rho * (u_1 * u_1); - const double xi_96 = xi_9 + xi_94 + xi_95; - const double xi_115 = rho * u_1; - const double xi_117 = - -vel1Term + xi_115 + xi_116 + xi_198 + xi_208 + xi_5; - const double xi_119 = xi_117 * xi_118; - const double xi_150 = - xi_149 * (u_0 * xi_115 + xi_116 + xi_204 + xi_8); - const double xi_155 = xi_117 * xi_154; - const double xi_156 = xi_155; - const double xi_157 = xi_113 + xi_156; - const double xi_166 = -xi_155; - const double xi_167 = xi_114 + xi_166; - const double xi_182 = xi_156 - xi_180 + xi_181; - const double xi_183 = xi_166 + xi_180 - xi_181; - const double u_2 = xi_7 * (vel2Term + xi_206 + xi_21 + xi_24); - const double xi_29 = u_2 * xi_218; - const double xi_30 = xi_29 * 0.333333333333333; - const double xi_31 = (-omega_bulk + 2.0) * (xi_26 + xi_28 + xi_30); - const double xi_34 = xi_29 * 0.666666666666667 + xi_32 + xi_33; - const double xi_37 = -xi_30; - const double xi_38 = xi_27 * 0.666666666666667 + xi_32 + xi_37; - const double xi_39 = xi_25 * 0.666666666666667 + xi_33 + xi_37; - const double xi_42 = xi_34 * xi_41; - const double xi_43 = -xi_42; - const double xi_44 = xi_39 * xi_41; - const double xi_45 = -xi_44; - const double xi_47 = xi_38 * xi_46 + xi_43 + xi_45; - const double xi_49 = xi_38 * xi_41; - const double xi_50 = -xi_49; - const double xi_51 = xi_39 * xi_46 + xi_43 + xi_50; - const double xi_53 = xi_34 * xi_46 + xi_45 + xi_50; - const double xi_60 = xi_44 - xi_59; - const double xi_62 = -xi_34 * xi_61; - const double xi_64 = xi_31 * 0.125; - const double xi_65 = xi_49 + xi_64; - const double xi_66 = xi_63 + xi_65; - const double xi_67 = xi_62 + xi_66; - const double xi_68 = xi_44 + xi_59; - const double xi_69 = -xi_63 + xi_65; - const double xi_70 = xi_62 + xi_69; - const double xi_71 = xi_56 * (u_2 * xi_55 + xi_218 * xi_54); - const double xi_72 = -xi_39 * xi_61; - const double xi_74 = xi_42 + xi_73; - const double xi_75 = xi_72 + xi_74; - const double xi_76 = -xi_71; - const double xi_77 = - xi_56 * (u_0 * xi_218 * 0.5 + u_2 * xi_211 * 0.5); - const double xi_78 = -xi_77; - const double xi_79 = -xi_38 * xi_61; - const double xi_80 = xi_64 + xi_74 + xi_79; - const double xi_81 = xi_42 - xi_73; - const double xi_82 = xi_72 + xi_81; - const double xi_83 = xi_64 + xi_79 + xi_81; - const double xi_91 = rho * (u_2 * u_2); - const double xi_99 = omega_bulk * (xi_17 + xi_202 + xi_22 + xi_90 + - xi_91 + xi_93 + xi_96 + xi_98); - const double xi_121 = xi_209 + xi_210 - xi_91; - const double xi_122 = - omega_shear * (xi_0 + xi_120 + xi_121 + xi_16 - xi_200 + xi_96); - const double xi_123 = xi_122 * 0.125; - const double xi_125 = - omega_shear * - (xi_121 + xi_200 + xi_203 * -2.0 + xi_207 * -2.0 + xi_219 + - xi_87 + xi_9 + xi_90 * 2.0 + xi_94 - xi_95 + xi_98); - const double xi_126 = xi_125 * -0.0416666666666667; - const double xi_127 = xi_126 + xi_86 * -0.05; - const double xi_128 = - xi_123 + xi_124 + xi_127 + xi_89 * 0.0142857142857143; - const double xi_137 = xi_124 + xi_125 * 0.0833333333333333 + - xi_89 * -0.0357142857142857; - const double xi_142 = - rho * u_2 - vel2Term + xi_139 + xi_212 + xi_6 + xi_92 + xi_97; - const double xi_143 = xi_118 * xi_142; - const double xi_147 = xi_103 * 0.0158730158730159 - xi_123 + xi_127 + - xi_89 * -0.0214285714285714; - const double xi_151 = xi_122 * 0.0625; - const double xi_153 = -xi_150 + xi_151 + xi_152; - const double xi_159 = xi_99 * 0.0416666666666667; - const double xi_160 = xi_125 * 0.0208333333333333 + xi_159; - const double xi_161 = -xi_158 + xi_160; - const double xi_162 = xi_138 + xi_161; - const double xi_163 = xi_150 + xi_151 + xi_152; - const double xi_164 = xi_158 + xi_160; - const double xi_165 = xi_136 + xi_164; - const double xi_169 = - xi_149 * (u_2 * xi_115 + xi_104 + xi_17 + xi_215); - const double xi_171 = xi_126 + xi_159 + xi_168 + xi_169 + xi_170; - const double xi_177 = xi_142 * xi_154; - const double xi_178 = xi_176 + xi_177; - const double xi_179 = -xi_173 + xi_175 + xi_178; - const double xi_184 = xi_126 + xi_159 + xi_168 - xi_169 + xi_170; - const double xi_185 = - xi_149 * (u_2 * xi_129 + xi_10 + xi_132 + xi_212); - const double xi_186 = -xi_151; - const double xi_188 = -xi_185 + xi_186 + xi_187; - const double xi_189 = xi_148 + xi_178; - const double xi_192 = xi_161 - xi_190 + xi_191; - const double xi_193 = xi_185 + xi_186 + xi_187; - const double xi_194 = xi_164 + xi_190 - xi_191; - const double xi_195 = xi_176 - xi_177; - const double xi_196 = xi_173 - xi_175 + xi_195; - const double xi_197 = xi_146 + xi_195; - const double forceTerm_0 = - xi_31 * -1.5 - xi_34 * xi_36 - xi_36 * xi_38 - xi_36 * xi_39; - const double forceTerm_1 = xi_40 + xi_47; - const double forceTerm_2 = -xi_40 + xi_47; - const double forceTerm_3 = -xi_48 + xi_51; - const double forceTerm_4 = xi_48 + xi_51; - const double forceTerm_5 = xi_52 + xi_53; - const double forceTerm_6 = -xi_52 + xi_53; - const double forceTerm_7 = xi_58 + xi_60 + xi_67; - const double forceTerm_8 = xi_57 + xi_67 + xi_68; - const double forceTerm_9 = xi_57 + xi_60 + xi_70; - const double forceTerm_10 = xi_58 + xi_68 + xi_70; - const double forceTerm_11 = xi_66 + xi_71 + xi_75; - const double forceTerm_12 = xi_69 + xi_75 + xi_76; - const double forceTerm_13 = xi_60 + xi_78 + xi_80; - const double forceTerm_14 = xi_68 + xi_77 + xi_80; - const double forceTerm_15 = xi_66 + xi_76 + xi_82; - const double forceTerm_16 = xi_69 + xi_71 + xi_82; - const double forceTerm_17 = xi_60 + xi_77 + xi_83; - const double forceTerm_18 = xi_68 + xi_78 + xi_83; - _data_pdfs_20_30_10[ctr_0] = - forceTerm_0 + xi_103 * 0.0238095238095238 + xi_202 + xi_86 * 0.1 + - xi_89 * 0.0428571428571429 + xi_99 * -0.5; - _data_pdfs_20_31_10[ctr_0] = - forceTerm_1 - xi_107 + xi_114 + xi_119 + xi_128 + xi_200; - _data_pdfs_20_32_10[ctr_0] = - forceTerm_2 + xi_107 + xi_113 - xi_119 + xi_128 + xi_219; - _data_pdfs_20_33_10[ctr_0] = - forceTerm_3 - xi_131 + xi_134 + xi_136 + xi_137 + xi_207; - _data_pdfs_20_34_10[ctr_0] = - forceTerm_4 + xi_131 - xi_134 + xi_137 + xi_138 + xi_203; - _data_pdfs_20_35_10[ctr_0] = - forceTerm_5 - xi_141 + xi_143 + xi_146 + xi_147 + xi_209; - _data_pdfs_20_36_10[ctr_0] = - forceTerm_6 + xi_141 - xi_143 + xi_147 + xi_148 + xi_210; - _data_pdfs_20_37_10[ctr_0] = - forceTerm_7 + xi_153 + xi_157 + xi_162 + xi_204; - _data_pdfs_20_38_10[ctr_0] = - forceTerm_8 + xi_157 + xi_163 + xi_165 + xi_213; - _data_pdfs_20_39_10[ctr_0] = - forceTerm_9 + xi_162 + xi_163 + xi_167 + xi_208; - _data_pdfs_20_310_10[ctr_0] = - forceTerm_10 + xi_153 + xi_165 + xi_167 + xi_217; - _data_pdfs_20_311_10[ctr_0] = - forceTerm_11 + xi_171 + xi_179 + xi_182 + xi_214; - _data_pdfs_20_312_10[ctr_0] = - forceTerm_12 + xi_179 + xi_183 + xi_184 + xi_198; - _data_pdfs_20_313_10[ctr_0] = - forceTerm_13 + xi_188 + xi_189 + xi_192 + xi_201; - _data_pdfs_20_314_10[ctr_0] = - forceTerm_14 + xi_189 + xi_193 + xi_194 + xi_206; - _data_pdfs_20_315_10[ctr_0] = - forceTerm_15 + xi_182 + xi_184 + xi_196 + xi_215; - _data_pdfs_20_316_10[ctr_0] = - forceTerm_16 + xi_171 + xi_183 + xi_196 + xi_199; - _data_pdfs_20_317_10[ctr_0] = - forceTerm_17 + xi_192 + xi_193 + xi_197 + xi_216; - _data_pdfs_20_318_10[ctr_0] = - forceTerm_18 + xi_188 + xi_194 + xi_197 + xi_212; - } + double * RESTRICT _data_pdfs_20_32_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_20_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_314; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_20_316_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_316; + double * RESTRICT _data_pdfs_20_38_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_20_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_36; + double * RESTRICT _data_pdfs_20_311_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_311; + double * RESTRICT _data_pdfs_20_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_318; + double * RESTRICT _data_pdfs_20_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_313; + double * RESTRICT _data_force_20_30_10 = _stride_force_1*ctr_1 + _data_force_20_30; + double * RESTRICT _data_pdfs_20_39_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_20_315_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_315; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_force_20_31_10 = _stride_force_1*ctr_1 + _data_force_20_31; + double * RESTRICT _data_pdfs_20_310_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_20_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_35; + double * RESTRICT _data_force_20_32_10 = _stride_force_1*ctr_1 + _data_force_20_32; + double * RESTRICT _data_pdfs_20_31_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_20_312_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_312; + double * RESTRICT _data_pdfs_20_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_20_37_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_37; + { + for (int64_t ctr_0 = 0; ctr_0 < (int64_t)((_size_force_0) / (4)) * (4); ctr_0 += 4) + { + const __m256d xi_207 = _mm256_loadu_pd(& _data_pdfs_20_32_10[ctr_0]); + const __m256d xi_208 = _mm256_loadu_pd(& _data_pdfs_20_314_10[ctr_0]); + const __m256d xi_209 = _mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0]); + const __m256d xi_210 = _mm256_loadu_pd(& _data_pdfs_20_316_10[ctr_0]); + const __m256d xi_211 = _mm256_loadu_pd(& _data_pdfs_20_38_10[ctr_0]); + const __m256d xi_212 = _mm256_loadu_pd(& _data_pdfs_20_36_10[ctr_0]); + const __m256d xi_213 = _mm256_loadu_pd(& _data_pdfs_20_311_10[ctr_0]); + const __m256d xi_214 = _mm256_loadu_pd(& _data_pdfs_20_318_10[ctr_0]); + const __m256d xi_215 = _mm256_loadu_pd(& _data_pdfs_20_313_10[ctr_0]); + const __m256d xi_216 = _mm256_loadu_pd(& _data_force_20_30_10[ctr_0]); + const __m256d xi_217 = _mm256_loadu_pd(& _data_pdfs_20_39_10[ctr_0]); + const __m256d xi_218 = _mm256_loadu_pd(& _data_pdfs_20_315_10[ctr_0]); + const __m256d xi_219 = _mm256_loadu_pd(& _data_pdfs_20_30_10[ctr_0]); + const __m256d xi_220 = _mm256_loadu_pd(& _data_force_20_31_10[ctr_0]); + const __m256d xi_221 = _mm256_loadu_pd(& _data_pdfs_20_310_10[ctr_0]); + const __m256d xi_222 = _mm256_loadu_pd(& _data_pdfs_20_35_10[ctr_0]); + const __m256d xi_223 = _mm256_loadu_pd(& _data_force_20_32_10[ctr_0]); + const __m256d xi_224 = _mm256_loadu_pd(& _data_pdfs_20_31_10[ctr_0]); + const __m256d xi_225 = _mm256_loadu_pd(& _data_pdfs_20_312_10[ctr_0]); + const __m256d xi_226 = _mm256_loadu_pd(& _data_pdfs_20_317_10[ctr_0]); + const __m256d xi_227 = _mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0]); + const __m256d xi_228 = _mm256_loadu_pd(& _data_pdfs_20_37_10[ctr_0]); + const __m256d xi_0 = _mm256_add_pd(xi_211,xi_221); + const __m256d xi_1 = _mm256_add_pd(xi_208,xi_214); + const __m256d xi_2 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_1),xi_209); + const __m256d xi_3 = _mm256_add_pd(xi_213,xi_218); + const __m256d xi_4 = _mm256_add_pd(xi_224,xi_3); + const __m256d xi_5 = _mm256_add_pd(xi_228,xi_4); + const __m256d xi_6 = _mm256_add_pd(xi_222,xi_225); + const __m256d xi_7 = _mm256_add_pd(xi_215,xi_6); + const __m256d xi_8 = _mm256_add_pd(xi_217,xi_227); + const __m256d xi_9 = _mm256_add_pd(xi_207,xi_210); + const __m256d xi_10 = _mm256_add_pd(xi_212,xi_226); + const __m256d xi_12 = _mm256_mul_pd(xi_226,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_13 = _mm256_mul_pd(xi_227,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_14 = _mm256_add_pd(xi_12,xi_13); + const __m256d xi_15 = _mm256_mul_pd(xi_215,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_16 = _mm256_mul_pd(xi_228,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_17 = _mm256_mul_pd(xi_217,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_18 = _mm256_add_pd(xi_16,xi_17); + const __m256d xi_19 = _mm256_add_pd(xi_15,xi_18); + const __m256d xi_20 = _mm256_mul_pd(xi_221,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_21 = _mm256_add_pd(xi_20,xi_211); + const __m256d xi_22 = _mm256_mul_pd(xi_225,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_23 = _mm256_mul_pd(xi_207,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_24 = _mm256_mul_pd(xi_210,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_25 = _mm256_add_pd(_mm256_add_pd(xi_22,xi_23),xi_24); + const __m256d xi_26 = _mm256_mul_pd(xi_214,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_27 = _mm256_add_pd(xi_12,xi_26); + const __m256d xi_28 = _mm256_mul_pd(xi_218,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_29 = _mm256_mul_pd(xi_212,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_30 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_213,xi_24),xi_28),xi_29); + const __m256d xi_31 = _mm256_add_pd(xi_215,xi_226); + const __m256d xi_32 = _mm256_add_pd(_mm256_add_pd(xi_228,xi_31),xi_8); + const __m256d xi_33 = _mm256_add_pd(_mm256_add_pd(xi_16,xi_21),xi_217); + const __m256d xi_34 = _mm256_add_pd(xi_226,xi_26); + const __m256d xi_35 = _mm256_add_pd(xi_15,xi_208); + const __m256d xi_36 = _mm256_add_pd(xi_34,xi_35); + const __m256d xi_37 = _mm256_add_pd(_mm256_add_pd(xi_217,xi_225),xi_9); + const __m256d xi_38 = _mm256_add_pd(xi_210,xi_28); + const __m256d xi_39 = _mm256_add_pd(_mm256_add_pd(xi_213,xi_22),xi_38); + const __m256d xi_55 = _mm256_mul_pd(xi_220,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_63 = _mm256_mul_pd(xi_216,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_67 = _mm256_mul_pd(xi_223,_mm256_set_pd(0.166666666666667,0.166666666666667,0.166666666666667,0.166666666666667)); + const __m256d xi_70 = _mm256_mul_pd(xi_220,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_74 = _mm256_mul_pd(xi_216,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_78 = _mm256_mul_pd(xi_220,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_88 = _mm256_mul_pd(xi_223,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333)); + const __m256d xi_99 = _mm256_mul_pd(xi_219,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_100 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_212,_mm256_set_pd(3.0,3.0,3.0,3.0)),_mm256_mul_pd(xi_222,_mm256_set_pd(3.0,3.0,3.0,3.0))),xi_99); + const __m256d xi_101 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_207,_mm256_set_pd(3.0,3.0,3.0,3.0)),_mm256_mul_pd(xi_210,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_213,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_218,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),_mm256_mul_pd(xi_224,_mm256_set_pd(3.0,3.0,3.0,3.0))),_mm256_mul_pd(xi_225,_mm256_set_pd(-3.0,-3.0,-3.0,-3.0))),xi_100),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_102 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_210,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_213,_mm256_set_pd(2.0,2.0,2.0,2.0))),_mm256_mul_pd(xi_218,_mm256_set_pd(2.0,2.0,2.0,2.0))),_mm256_mul_pd(xi_225,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_103 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_209,_mm256_set_pd(5.0,5.0,5.0,5.0)),_mm256_mul_pd(xi_227,_mm256_set_pd(5.0,5.0,5.0,5.0))),xi_102); + const __m256d xi_104 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_207,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0)),_mm256_mul_pd(xi_208,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_214,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_215,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),_mm256_mul_pd(xi_224,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),_mm256_mul_pd(xi_226,_mm256_set_pd(-5.0,-5.0,-5.0,-5.0))),xi_100),xi_103),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_107 = _mm256_mul_pd(xi_213,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_108 = _mm256_add_pd(xi_107,xi_22); + const __m256d xi_109 = _mm256_mul_pd(xi_211,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_112 = _mm256_mul_pd(xi_208,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_113 = _mm256_add_pd(_mm256_add_pd(xi_112,xi_19),xi_27); + const __m256d xi_115 = _mm256_mul_pd(xi_215,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_116 = _mm256_mul_pd(xi_208,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_117 = _mm256_add_pd(_mm256_mul_pd(xi_214,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_226,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_118 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_207,_mm256_set_pd(5.0,5.0,5.0,5.0)),_mm256_mul_pd(xi_211,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_212,_mm256_set_pd(-4.0,-4.0,-4.0,-4.0))),_mm256_mul_pd(xi_217,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_221,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),_mm256_mul_pd(xi_222,_mm256_set_pd(-4.0,-4.0,-4.0,-4.0))),_mm256_mul_pd(xi_224,_mm256_set_pd(5.0,5.0,5.0,5.0))),_mm256_mul_pd(xi_228,_mm256_set_pd(-7.0,-7.0,-7.0,-7.0))),xi_103),xi_115),xi_116),xi_117),xi_99),_mm256_set_pd(omega_even,omega_even,omega_even,omega_even)); + const __m256d xi_119 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_107,xi_224),xi_225),xi_23),xi_38); + const __m256d xi_121 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_122 = _mm256_mul_pd(xi_228,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_123 = _mm256_mul_pd(xi_221,_mm256_set_pd(2.0,2.0,2.0,2.0)); + const __m256d xi_124 = _mm256_add_pd(_mm256_mul_pd(xi_211,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0)),_mm256_mul_pd(xi_217,_mm256_set_pd(2.0,2.0,2.0,2.0))); + const __m256d xi_125 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_122,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_123),xi_124),xi_25),xi_4); + const __m256d xi_127 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_128 = _mm256_mul_pd(xi_127,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_133 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.0198412698412698,-0.0198412698412698,-0.0198412698412698,-0.0198412698412698)); + const __m256d xi_144 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_112,xi_13),xi_209),xi_215),xi_34); + const __m256d xi_145 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_146 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_123,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_122),xi_124),xi_14),xi_209),xi_214),xi_35); + const __m256d xi_147 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_149 = _mm256_mul_pd(xi_147,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_150 = _mm256_add_pd(xi_210,xi_218); + const __m256d xi_151 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_108,xi_150),xi_222),xi_29); + const __m256d xi_152 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_120,xi_120,xi_120,xi_120)); + const __m256d xi_153 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_115,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(xi_116,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_117),xi_30),xi_6); + const __m256d xi_154 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_126,xi_126,xi_126,xi_126)); + const __m256d xi_155 = _mm256_mul_pd(xi_154,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_162 = _mm256_mul_pd(xi_118,_mm256_set_pd(0.0138888888888889,0.0138888888888889,0.0138888888888889,0.0138888888888889)); + const __m256d xi_177 = _mm256_mul_pd(xi_104,_mm256_set_pd(-0.00714285714285714,-0.00714285714285714,-0.00714285714285714,-0.00714285714285714)); + const __m256d xi_179 = _mm256_mul_pd(xi_101,_mm256_set_pd(0.025,0.025,0.025,0.025)); + const __m256d xi_182 = _mm256_mul_pd(xi_153,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_184 = _mm256_mul_pd(xi_151,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_185 = _mm256_mul_pd(xi_118,_mm256_set_pd(-0.00396825396825397,-0.00396825396825397,-0.00396825396825397,-0.00396825396825397)); + const __m256d xi_189 = _mm256_mul_pd(xi_125,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d xi_190 = _mm256_mul_pd(xi_119,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_196 = _mm256_mul_pd(xi_104,_mm256_set_pd(0.0178571428571429,0.0178571428571429,0.0178571428571429,0.0178571428571429)); + const __m256d xi_199 = _mm256_mul_pd(xi_144,_mm256_set_pd(xi_183,xi_183,xi_183,xi_183)); + const __m256d xi_200 = _mm256_mul_pd(xi_146,_mm256_set_pd(xi_181,xi_181,xi_181,xi_181)); + const __m256d vel0Term = xi_2; + const __m256d vel1Term = xi_5; + const __m256d vel2Term = xi_7; + const __m256d rho = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel0Term,vel1Term),vel2Term),xi_10),xi_219),xi_8),xi_9); + const __m256d xi_11 = _mm256_div_pd(_mm256_set_pd(1.0,1.0,1.0,1.0),rho); + const __m256d u_0 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(vel0Term,xi_14),xi_19)); + const __m256d xi_40 = _mm256_mul_pd(u_0,xi_216); + const __m256d xi_41 = _mm256_mul_pd(xi_40,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_47 = _mm256_mul_pd(xi_41,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_105 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_0,u_0))); + const __m256d xi_141 = _mm256_mul_pd(rho,u_0); + const __m256d xi_142 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel0Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_141),xi_32); + const __m256d xi_143 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_167 = _mm256_mul_pd(xi_142,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d u_1 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel1Term,xi_17),xi_21),xi_25)); + const __m256d xi_42 = _mm256_mul_pd(u_1,xi_220); + const __m256d xi_43 = _mm256_mul_pd(xi_42,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_48 = _mm256_mul_pd(xi_43,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_69 = _mm256_mul_pd(u_1,_mm256_set_pd(0.5,0.5,0.5,0.5)); + const __m256d xi_72 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_0,xi_70),_mm256_mul_pd(xi_216,xi_69)),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_73 = _mm256_mul_pd(xi_72,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_110 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_1,u_1))); + const __m256d xi_111 = _mm256_add_pd(_mm256_add_pd(xi_109,xi_110),xi_20); + const __m256d xi_129 = _mm256_mul_pd(rho,u_1); + const __m256d xi_130 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(vel1Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_109),xi_129),xi_221),xi_37); + const __m256d xi_132 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_165 = _mm256_mul_pd(xi_130,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_166 = _mm256_add_pd(xi_127,xi_165); + const __m256d xi_175 = _mm256_mul_pd(xi_165,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_176 = _mm256_add_pd(xi_128,xi_175); + const __m256d xi_191 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_189,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_165),xi_190); + const __m256d xi_192 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_190,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_175),xi_189); + const __m256d u_2 = _mm256_mul_pd(xi_11,_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(vel2Term,xi_208),xi_27),xi_30)); + const __m256d xi_44 = _mm256_mul_pd(u_2,xi_223); + const __m256d xi_45 = _mm256_mul_pd(xi_44,_mm256_set_pd(0.333333333333333,0.333333333333333,0.333333333333333,0.333333333333333)); + const __m256d xi_46 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(xi_41,xi_43),xi_45),_mm256_set_pd(-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0,-omega_bulk + 2.0)); + const __m256d xi_49 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_44,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_48); + const __m256d xi_52 = _mm256_mul_pd(xi_45,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_53 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_42,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_47),xi_52); + const __m256d xi_54 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_40,_mm256_set_pd(0.666666666666667,0.666666666666667,0.666666666666667,0.666666666666667)),xi_48),xi_52); + const __m256d xi_57 = _mm256_mul_pd(xi_49,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_58 = _mm256_mul_pd(xi_57,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_59 = _mm256_mul_pd(xi_54,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_60 = _mm256_mul_pd(xi_59,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_62 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_60); + const __m256d xi_64 = _mm256_mul_pd(xi_53,_mm256_set_pd(xi_56,xi_56,xi_56,xi_56)); + const __m256d xi_65 = _mm256_mul_pd(xi_64,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_66 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_58),xi_65); + const __m256d xi_68 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(xi_61,xi_61,xi_61,xi_61)),xi_60),xi_65); + const __m256d xi_75 = _mm256_add_pd(_mm256_mul_pd(xi_74,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_59); + const __m256d xi_77 = _mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_79 = _mm256_mul_pd(xi_46,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_80 = _mm256_add_pd(xi_64,xi_79); + const __m256d xi_81 = _mm256_add_pd(xi_78,xi_80); + const __m256d xi_82 = _mm256_add_pd(xi_77,xi_81); + const __m256d xi_83 = _mm256_add_pd(xi_59,xi_74); + const __m256d xi_84 = _mm256_add_pd(_mm256_mul_pd(xi_78,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_80); + const __m256d xi_85 = _mm256_add_pd(xi_77,xi_84); + const __m256d xi_86 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(u_2,xi_70),_mm256_mul_pd(xi_223,xi_69)),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_87 = _mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_89 = _mm256_add_pd(xi_57,xi_88); + const __m256d xi_90 = _mm256_add_pd(xi_87,xi_89); + const __m256d xi_91 = _mm256_mul_pd(xi_86,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_92 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(_mm256_mul_pd(u_0,xi_223),_mm256_set_pd(0.5,0.5,0.5,0.5)),_mm256_mul_pd(_mm256_mul_pd(u_2,xi_216),_mm256_set_pd(0.5,0.5,0.5,0.5))),_mm256_set_pd(xi_71,xi_71,xi_71,xi_71)); + const __m256d xi_93 = _mm256_mul_pd(xi_92,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_94 = _mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_76,xi_76,xi_76,xi_76)); + const __m256d xi_95 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_89),xi_94); + const __m256d xi_96 = _mm256_add_pd(_mm256_mul_pd(xi_88,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_57); + const __m256d xi_97 = _mm256_add_pd(xi_87,xi_96); + const __m256d xi_98 = _mm256_add_pd(_mm256_add_pd(xi_79,xi_94),xi_96); + const __m256d xi_106 = _mm256_mul_pd(rho,(_mm256_mul_pd(u_2,u_2))); + const __m256d xi_114 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_105,xi_106),xi_108),xi_111),xi_113),xi_219),xi_24),xi_28),_mm256_set_pd(omega_bulk,omega_bulk,omega_bulk,omega_bulk)); + const __m256d xi_134 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_106,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_212),xi_222); + const __m256d xi_135 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_224,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_1),xi_111),xi_134),xi_18),xi_23),xi_31),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_136 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.125,0.125,0.125,0.125)); + const __m256d xi_137 = _mm256_mul_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_105,_mm256_set_pd(2.0,2.0,2.0,2.0)),_mm256_mul_pd(xi_110,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),_mm256_mul_pd(xi_209,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),_mm256_mul_pd(xi_227,_mm256_set_pd(-2.0,-2.0,-2.0,-2.0))),xi_102),xi_109),xi_113),xi_134),xi_20),xi_207),xi_224),_mm256_set_pd(omega_shear,omega_shear,omega_shear,omega_shear)); + const __m256d xi_138 = _mm256_mul_pd(xi_137,_mm256_set_pd(-0.0416666666666667,-0.0416666666666667,-0.0416666666666667,-0.0416666666666667)); + const __m256d xi_139 = _mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(-0.05,-0.05,-0.05,-0.05)),xi_138); + const __m256d xi_140 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(0.0142857142857143,0.0142857142857143,0.0142857142857143,0.0142857142857143)),xi_133),xi_136),xi_139); + const __m256d xi_148 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0357142857142857,-0.0357142857142857,-0.0357142857142857,-0.0357142857142857)),_mm256_mul_pd(xi_137,_mm256_set_pd(0.0833333333333333,0.0833333333333333,0.0833333333333333,0.0833333333333333))),xi_133); + const __m256d xi_156 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(rho,u_2),_mm256_mul_pd(vel2Term,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_10),xi_107),xi_112),xi_150),xi_214); + const __m256d xi_157 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_131,xi_131,xi_131,xi_131)); + const __m256d xi_158 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_104,_mm256_set_pd(-0.0214285714285714,-0.0214285714285714,-0.0214285714285714,-0.0214285714285714)),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0158730158730159,0.0158730158730159,0.0158730158730159,0.0158730158730159))),_mm256_mul_pd(xi_136,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0))),xi_139); + const __m256d xi_161 = _mm256_mul_pd(xi_135,_mm256_set_pd(0.0625,0.0625,0.0625,0.0625)); + const __m256d xi_168 = _mm256_mul_pd(xi_114,_mm256_set_pd(0.0416666666666667,0.0416666666666667,0.0416666666666667,0.0416666666666667)); + const __m256d xi_169 = _mm256_add_pd(_mm256_mul_pd(xi_137,_mm256_set_pd(0.0208333333333333,0.0208333333333333,0.0208333333333333,0.0208333333333333)),xi_168); + const __m256d xi_170 = _mm256_add_pd(_mm256_mul_pd(xi_167,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_169); + const __m256d xi_171 = _mm256_add_pd(xi_149,xi_170); + const __m256d xi_173 = _mm256_add_pd(xi_167,xi_169); + const __m256d xi_174 = _mm256_add_pd(xi_147,xi_173); + const __m256d xi_186 = _mm256_mul_pd(xi_156,_mm256_set_pd(xi_164,xi_164,xi_164,xi_164)); + const __m256d xi_187 = _mm256_add_pd(xi_185,xi_186); + const __m256d xi_188 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_182,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_184),xi_187); + const __m256d xi_195 = _mm256_mul_pd(xi_161,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)); + const __m256d xi_198 = _mm256_add_pd(xi_154,xi_187); + const __m256d xi_201 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_199,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_170),xi_200); + const __m256d xi_203 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_200,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_173),xi_199); + const __m256d xi_204 = _mm256_add_pd(_mm256_mul_pd(xi_186,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_185); + const __m256d xi_205 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_184,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_182),xi_204); + const __m256d xi_206 = _mm256_add_pd(xi_155,xi_204); + const __m256d p_0 = _mm256_add_pd(xi_2,xi_32); + const __m256d p_1 = xi_33; + const __m256d xi_160 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_1,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_0,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_163 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_160,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_161),xi_162); + const __m256d xi_172 = _mm256_add_pd(_mm256_add_pd(xi_160,xi_161),xi_162); + const __m256d p_2 = xi_36; + const __m256d xi_194 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_2,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_141)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_197 = _mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_194,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_195),xi_196); + const __m256d xi_202 = _mm256_add_pd(_mm256_add_pd(xi_194,xi_195),xi_196); + const __m256d p_3 = xi_33; + const __m256d p_4 = _mm256_add_pd(_mm256_add_pd(xi_0,xi_37),xi_5); + const __m256d p_5 = xi_39; + const __m256d xi_178 = _mm256_mul_pd(_mm256_add_pd(_mm256_mul_pd(p_5,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_mul_pd(u_2,xi_129)),_mm256_set_pd(xi_159,xi_159,xi_159,xi_159)); + const __m256d xi_180 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_138,xi_168),xi_177),xi_178),xi_179); + const __m256d xi_193 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_178,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_138),xi_168),xi_177),xi_179); + const __m256d p_6 = xi_36; + const __m256d p_7 = xi_39; + const __m256d p_8 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(xi_1,xi_10),xi_210),xi_3),xi_7); + const __m256d forceTerm_0 = _mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_46,_mm256_set_pd(-1.5,-1.5,-1.5,-1.5)),_mm256_mul_pd(_mm256_mul_pd(xi_49,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_53,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))),_mm256_mul_pd(_mm256_mul_pd(xi_54,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),_mm256_set_pd(xi_51,xi_51,xi_51,xi_51))); + const __m256d forceTerm_1 = _mm256_add_pd(xi_55,xi_62); + const __m256d forceTerm_2 = _mm256_add_pd(_mm256_mul_pd(xi_55,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_62); + const __m256d forceTerm_3 = _mm256_add_pd(_mm256_mul_pd(xi_63,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_66); + const __m256d forceTerm_4 = _mm256_add_pd(xi_63,xi_66); + const __m256d forceTerm_5 = _mm256_add_pd(xi_67,xi_68); + const __m256d forceTerm_6 = _mm256_add_pd(_mm256_mul_pd(xi_67,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),xi_68); + const __m256d forceTerm_7 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_75),xi_82); + const __m256d forceTerm_8 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_82),xi_83); + const __m256d forceTerm_9 = _mm256_add_pd(_mm256_add_pd(xi_72,xi_75),xi_85); + const __m256d forceTerm_10 = _mm256_add_pd(_mm256_add_pd(xi_73,xi_83),xi_85); + const __m256d forceTerm_11 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_86),xi_90); + const __m256d forceTerm_12 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_90),xi_91); + const __m256d forceTerm_13 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_93),xi_95); + const __m256d forceTerm_14 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_92),xi_95); + const __m256d forceTerm_15 = _mm256_add_pd(_mm256_add_pd(xi_81,xi_91),xi_97); + const __m256d forceTerm_16 = _mm256_add_pd(_mm256_add_pd(xi_84,xi_86),xi_97); + const __m256d forceTerm_17 = _mm256_add_pd(_mm256_add_pd(xi_75,xi_92),xi_98); + const __m256d forceTerm_18 = _mm256_add_pd(_mm256_add_pd(xi_83,xi_93),xi_98); + _mm256_storeu_pd(&_data_pdfs_20_30_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_101,_mm256_set_pd(0.1,0.1,0.1,0.1)),_mm256_mul_pd(xi_104,_mm256_set_pd(0.0428571428571429,0.0428571428571429,0.0428571428571429,0.0428571428571429))),_mm256_mul_pd(xi_114,_mm256_set_pd(-0.5,-0.5,-0.5,-0.5))),_mm256_mul_pd(xi_118,_mm256_set_pd(0.0238095238095238,0.0238095238095238,0.0238095238095238,0.0238095238095238))),forceTerm_0),xi_219)); + _mm256_storeu_pd(&_data_pdfs_20_31_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_121,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_1),xi_128),xi_132),xi_140),xi_224)); + _mm256_storeu_pd(&_data_pdfs_20_32_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_132,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_2),xi_121),xi_127),xi_140),xi_207)); + _mm256_storeu_pd(&_data_pdfs_20_33_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_143,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_3),xi_145),xi_147),xi_148),xi_227)); + _mm256_storeu_pd(&_data_pdfs_20_34_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_145,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_4),xi_143),xi_148),xi_149),xi_209)); + _mm256_storeu_pd(&_data_pdfs_20_35_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_152,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_5),xi_155),xi_157),xi_158),xi_222)); + _mm256_storeu_pd(&_data_pdfs_20_36_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_mul_pd(xi_157,_mm256_set_pd(-1.0,-1.0,-1.0,-1.0)),forceTerm_6),xi_152),xi_154),xi_158),xi_212)); + _mm256_storeu_pd(&_data_pdfs_20_37_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_7,xi_163),xi_166),xi_171),xi_228)); + _mm256_storeu_pd(&_data_pdfs_20_38_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_8,xi_166),xi_172),xi_174),xi_211)); + _mm256_storeu_pd(&_data_pdfs_20_39_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_9,xi_171),xi_172),xi_176),xi_217)); + _mm256_storeu_pd(&_data_pdfs_20_310_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_10,xi_163),xi_174),xi_176),xi_221)); + _mm256_storeu_pd(&_data_pdfs_20_311_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_11,xi_180),xi_188),xi_191),xi_213)); + _mm256_storeu_pd(&_data_pdfs_20_312_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_12,xi_188),xi_192),xi_193),xi_225)); + _mm256_storeu_pd(&_data_pdfs_20_313_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_13,xi_197),xi_198),xi_201),xi_215)); + _mm256_storeu_pd(&_data_pdfs_20_314_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_14,xi_198),xi_202),xi_203),xi_208)); + _mm256_storeu_pd(&_data_pdfs_20_315_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_15,xi_191),xi_193),xi_205),xi_218)); + _mm256_storeu_pd(&_data_pdfs_20_316_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_16,xi_180),xi_192),xi_205),xi_210)); + _mm256_storeu_pd(&_data_pdfs_20_317_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_17,xi_201),xi_202),xi_206),xi_226)); + _mm256_storeu_pd(&_data_pdfs_20_318_10[ctr_0],_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(_mm256_add_pd(forceTerm_18,xi_197),xi_203),xi_206),xi_214)); + } + for (int64_t ctr_0 = (int64_t)((_size_force_0) / (4)) * (4); ctr_0 < _size_force_0; ctr_0 += 1) + { + const double xi_207 = _data_pdfs_20_32_10[ctr_0]; + const double xi_208 = _data_pdfs_20_314_10[ctr_0]; + const double xi_209 = _data_pdfs_20_34_10[ctr_0]; + const double xi_210 = _data_pdfs_20_316_10[ctr_0]; + const double xi_211 = _data_pdfs_20_38_10[ctr_0]; + const double xi_212 = _data_pdfs_20_36_10[ctr_0]; + const double xi_213 = _data_pdfs_20_311_10[ctr_0]; + const double xi_214 = _data_pdfs_20_318_10[ctr_0]; + const double xi_215 = _data_pdfs_20_313_10[ctr_0]; + const double xi_216 = _data_force_20_30_10[ctr_0]; + const double xi_217 = _data_pdfs_20_39_10[ctr_0]; + const double xi_218 = _data_pdfs_20_315_10[ctr_0]; + const double xi_219 = _data_pdfs_20_30_10[ctr_0]; + const double xi_220 = _data_force_20_31_10[ctr_0]; + const double xi_221 = _data_pdfs_20_310_10[ctr_0]; + const double xi_222 = _data_pdfs_20_35_10[ctr_0]; + const double xi_223 = _data_force_20_32_10[ctr_0]; + const double xi_224 = _data_pdfs_20_31_10[ctr_0]; + const double xi_225 = _data_pdfs_20_312_10[ctr_0]; + const double xi_226 = _data_pdfs_20_317_10[ctr_0]; + const double xi_227 = _data_pdfs_20_33_10[ctr_0]; + const double xi_228 = _data_pdfs_20_37_10[ctr_0]; + const double xi_0 = xi_211 + xi_221; + const double xi_1 = xi_208 + xi_214; + const double xi_2 = xi_0 + xi_1 + xi_209; + const double xi_3 = xi_213 + xi_218; + const double xi_4 = xi_224 + xi_3; + const double xi_5 = xi_228 + xi_4; + const double xi_6 = xi_222 + xi_225; + const double xi_7 = xi_215 + xi_6; + const double xi_8 = xi_217 + xi_227; + const double xi_9 = xi_207 + xi_210; + const double xi_10 = xi_212 + xi_226; + const double xi_12 = -xi_226; + const double xi_13 = -xi_227; + const double xi_14 = xi_12 + xi_13; + const double xi_15 = -xi_215; + const double xi_16 = -xi_228; + const double xi_17 = -xi_217; + const double xi_18 = xi_16 + xi_17; + const double xi_19 = xi_15 + xi_18; + const double xi_20 = -xi_221; + const double xi_21 = xi_20 + xi_211; + const double xi_22 = -xi_225; + const double xi_23 = -xi_207; + const double xi_24 = -xi_210; + const double xi_25 = xi_22 + xi_23 + xi_24; + const double xi_26 = -xi_214; + const double xi_27 = xi_12 + xi_26; + const double xi_28 = -xi_218; + const double xi_29 = -xi_212; + const double xi_30 = xi_213 + xi_24 + xi_28 + xi_29; + const double xi_31 = xi_215 + xi_226; + const double xi_32 = xi_228 + xi_31 + xi_8; + const double xi_33 = xi_16 + xi_21 + xi_217; + const double xi_34 = xi_226 + xi_26; + const double xi_35 = xi_15 + xi_208; + const double xi_36 = xi_34 + xi_35; + const double xi_37 = xi_217 + xi_225 + xi_9; + const double xi_38 = xi_210 + xi_28; + const double xi_39 = xi_213 + xi_22 + xi_38; + const double xi_55 = xi_220*0.166666666666667; + const double xi_63 = xi_216*0.166666666666667; + const double xi_67 = xi_223*0.166666666666667; + const double xi_70 = xi_220*0.5; + const double xi_74 = xi_216*0.0833333333333333; + const double xi_78 = xi_220*0.0833333333333333; + const double xi_88 = xi_223*0.0833333333333333; + const double xi_99 = -xi_219; + const double xi_100 = xi_212*3.0 + xi_222*3.0 + xi_99; + const double xi_101 = omega_even*(xi_100 + xi_207*3.0 + xi_210*-3.0 + xi_213*-3.0 + xi_218*-3.0 + xi_224*3.0 + xi_225*-3.0); + const double xi_102 = xi_210*2.0 + xi_213*2.0 + xi_218*2.0 + xi_225*2.0; + const double xi_103 = xi_102 + xi_209*5.0 + xi_227*5.0; + const double xi_104 = omega_even*(xi_100 + xi_103 + xi_207*-2.0 + xi_208*-5.0 + xi_214*-5.0 + xi_215*-5.0 + xi_224*-2.0 + xi_226*-5.0); + const double xi_107 = -xi_213; + const double xi_108 = xi_107 + xi_22; + const double xi_109 = -xi_211; + const double xi_112 = -xi_208; + const double xi_113 = xi_112 + xi_19 + xi_27; + const double xi_115 = xi_215*2.0; + const double xi_116 = xi_208*2.0; + const double xi_117 = xi_214*2.0 + xi_226*2.0; + const double xi_118 = omega_even*(xi_103 + xi_115 + xi_116 + xi_117 + xi_207*5.0 + xi_211*-7.0 + xi_212*-4.0 + xi_217*-7.0 + xi_221*-7.0 + xi_222*-4.0 + xi_224*5.0 + xi_228*-7.0 + xi_99); + const double xi_119 = xi_107 + xi_224 + xi_225 + xi_23 + xi_38; + const double xi_121 = xi_119*xi_120; + const double xi_122 = xi_228*2.0; + const double xi_123 = xi_221*2.0; + const double xi_124 = xi_211*-2.0 + xi_217*2.0; + const double xi_125 = -xi_122 + xi_123 + xi_124 + xi_25 + xi_4; + const double xi_127 = xi_125*xi_126; + const double xi_128 = -xi_127; + const double xi_133 = xi_118*-0.0198412698412698; + const double xi_144 = xi_112 + xi_13 + xi_209 + xi_215 + xi_34; + const double xi_145 = xi_120*xi_144; + const double xi_146 = xi_122 - xi_123 + xi_124 + xi_14 + xi_209 + xi_214 + xi_35; + const double xi_147 = xi_126*xi_146; + const double xi_149 = -xi_147; + const double xi_150 = xi_210 + xi_218; + const double xi_151 = xi_108 + xi_150 + xi_222 + xi_29; + const double xi_152 = xi_120*xi_151; + const double xi_153 = -xi_115 - xi_116 + xi_117 + xi_30 + xi_6; + const double xi_154 = xi_126*xi_153; + const double xi_155 = -xi_154; + const double xi_162 = xi_118*0.0138888888888889; + const double xi_177 = xi_104*-0.00714285714285714; + const double xi_179 = xi_101*0.025; + const double xi_182 = xi_153*xi_181; + const double xi_184 = xi_151*xi_183; + const double xi_185 = xi_118*-0.00396825396825397; + const double xi_189 = xi_125*xi_181; + const double xi_190 = xi_119*xi_183; + const double xi_196 = xi_104*0.0178571428571429; + const double xi_199 = xi_144*xi_183; + const double xi_200 = xi_146*xi_181; + const double vel0Term = xi_2; + const double vel1Term = xi_5; + const double vel2Term = xi_7; + const double rho = vel0Term + vel1Term + vel2Term + xi_10 + xi_219 + xi_8 + xi_9; + const double xi_11 = 1 / (rho); + const double u_0 = xi_11*(vel0Term + xi_14 + xi_19); + const double xi_40 = u_0*xi_216; + const double xi_41 = xi_40*0.333333333333333; + const double xi_47 = -xi_41; + const double xi_105 = rho*(u_0*u_0); + const double xi_141 = rho*u_0; + const double xi_142 = -vel0Term + xi_141 + xi_32; + const double xi_143 = xi_131*xi_142; + const double xi_167 = xi_142*xi_164; + const double u_1 = xi_11*(vel1Term + xi_17 + xi_21 + xi_25); + const double xi_42 = u_1*xi_220; + const double xi_43 = xi_42*0.333333333333333; + const double xi_48 = -xi_43; + const double xi_69 = u_1*0.5; + const double xi_72 = xi_71*(u_0*xi_70 + xi_216*xi_69); + const double xi_73 = -xi_72; + const double xi_110 = rho*(u_1*u_1); + const double xi_111 = xi_109 + xi_110 + xi_20; + const double xi_129 = rho*u_1; + const double xi_130 = -vel1Term + xi_109 + xi_129 + xi_221 + xi_37; + const double xi_132 = xi_130*xi_131; + const double xi_165 = xi_130*xi_164; + const double xi_166 = xi_127 + xi_165; + const double xi_175 = -xi_165; + const double xi_176 = xi_128 + xi_175; + const double xi_191 = xi_165 - xi_189 + xi_190; + const double xi_192 = xi_175 + xi_189 - xi_190; + const double u_2 = xi_11*(vel2Term + xi_208 + xi_27 + xi_30); + const double xi_44 = u_2*xi_223; + const double xi_45 = xi_44*0.333333333333333; + const double xi_46 = (-omega_bulk + 2.0)*(xi_41 + xi_43 + xi_45); + const double xi_49 = xi_44*0.666666666666667 + xi_47 + xi_48; + const double xi_52 = -xi_45; + const double xi_53 = xi_42*0.666666666666667 + xi_47 + xi_52; + const double xi_54 = xi_40*0.666666666666667 + xi_48 + xi_52; + const double xi_57 = xi_49*xi_56; + const double xi_58 = -xi_57; + const double xi_59 = xi_54*xi_56; + const double xi_60 = -xi_59; + const double xi_62 = xi_53*xi_61 + xi_58 + xi_60; + const double xi_64 = xi_53*xi_56; + const double xi_65 = -xi_64; + const double xi_66 = xi_54*xi_61 + xi_58 + xi_65; + const double xi_68 = xi_49*xi_61 + xi_60 + xi_65; + const double xi_75 = xi_59 - xi_74; + const double xi_77 = -xi_49*xi_76; + const double xi_79 = xi_46*0.125; + const double xi_80 = xi_64 + xi_79; + const double xi_81 = xi_78 + xi_80; + const double xi_82 = xi_77 + xi_81; + const double xi_83 = xi_59 + xi_74; + const double xi_84 = -xi_78 + xi_80; + const double xi_85 = xi_77 + xi_84; + const double xi_86 = xi_71*(u_2*xi_70 + xi_223*xi_69); + const double xi_87 = -xi_54*xi_76; + const double xi_89 = xi_57 + xi_88; + const double xi_90 = xi_87 + xi_89; + const double xi_91 = -xi_86; + const double xi_92 = xi_71*(u_0*xi_223*0.5 + u_2*xi_216*0.5); + const double xi_93 = -xi_92; + const double xi_94 = -xi_53*xi_76; + const double xi_95 = xi_79 + xi_89 + xi_94; + const double xi_96 = xi_57 - xi_88; + const double xi_97 = xi_87 + xi_96; + const double xi_98 = xi_79 + xi_94 + xi_96; + const double xi_106 = rho*(u_2*u_2); + const double xi_114 = omega_bulk*(xi_105 + xi_106 + xi_108 + xi_111 + xi_113 + xi_219 + xi_24 + xi_28); + const double xi_134 = -xi_106 + xi_212 + xi_222; + const double xi_135 = omega_shear*(xi_1 + xi_111 + xi_134 + xi_18 - xi_224 + xi_23 + xi_31); + const double xi_136 = xi_135*0.125; + const double xi_137 = omega_shear*(xi_102 + xi_105*2.0 + xi_109 - xi_110 + xi_113 + xi_134 + xi_20 + xi_207 + xi_209*-2.0 + xi_224 + xi_227*-2.0); + const double xi_138 = xi_137*-0.0416666666666667; + const double xi_139 = xi_101*-0.05 + xi_138; + const double xi_140 = xi_104*0.0142857142857143 + xi_133 + xi_136 + xi_139; + const double xi_148 = xi_104*-0.0357142857142857 + xi_133 + xi_137*0.0833333333333333; + const double xi_156 = rho*u_2 - vel2Term + xi_10 + xi_107 + xi_112 + xi_150 + xi_214; + const double xi_157 = xi_131*xi_156; + const double xi_158 = xi_104*-0.0214285714285714 + xi_118*0.0158730158730159 - xi_136 + xi_139; + const double xi_161 = xi_135*0.0625; + const double xi_168 = xi_114*0.0416666666666667; + const double xi_169 = xi_137*0.0208333333333333 + xi_168; + const double xi_170 = -xi_167 + xi_169; + const double xi_171 = xi_149 + xi_170; + const double xi_173 = xi_167 + xi_169; + const double xi_174 = xi_147 + xi_173; + const double xi_186 = xi_156*xi_164; + const double xi_187 = xi_185 + xi_186; + const double xi_188 = -xi_182 + xi_184 + xi_187; + const double xi_195 = -xi_161; + const double xi_198 = xi_154 + xi_187; + const double xi_201 = xi_170 - xi_199 + xi_200; + const double xi_203 = xi_173 + xi_199 - xi_200; + const double xi_204 = xi_185 - xi_186; + const double xi_205 = xi_182 - xi_184 + xi_204; + const double xi_206 = xi_155 + xi_204; + const double p_0 = xi_2 + xi_32; + const double p_1 = xi_33; + const double xi_160 = xi_159*(-p_1 + u_0*xi_129); + const double xi_163 = -xi_160 + xi_161 + xi_162; + const double xi_172 = xi_160 + xi_161 + xi_162; + const double p_2 = xi_36; + const double xi_194 = xi_159*(-p_2 + u_2*xi_141); + const double xi_197 = -xi_194 + xi_195 + xi_196; + const double xi_202 = xi_194 + xi_195 + xi_196; + const double p_3 = xi_33; + const double p_4 = xi_0 + xi_37 + xi_5; + const double p_5 = xi_39; + const double xi_178 = xi_159*(-p_5 + u_2*xi_129); + const double xi_180 = xi_138 + xi_168 + xi_177 + xi_178 + xi_179; + const double xi_193 = xi_138 + xi_168 + xi_177 - xi_178 + xi_179; + const double p_6 = xi_36; + const double p_7 = xi_39; + const double p_8 = xi_1 + xi_10 + xi_210 + xi_3 + xi_7; + const double forceTerm_0 = xi_46*-1.5 - xi_49*xi_51 - xi_51*xi_53 - xi_51*xi_54; + const double forceTerm_1 = xi_55 + xi_62; + const double forceTerm_2 = -xi_55 + xi_62; + const double forceTerm_3 = -xi_63 + xi_66; + const double forceTerm_4 = xi_63 + xi_66; + const double forceTerm_5 = xi_67 + xi_68; + const double forceTerm_6 = -xi_67 + xi_68; + const double forceTerm_7 = xi_73 + xi_75 + xi_82; + const double forceTerm_8 = xi_72 + xi_82 + xi_83; + const double forceTerm_9 = xi_72 + xi_75 + xi_85; + const double forceTerm_10 = xi_73 + xi_83 + xi_85; + const double forceTerm_11 = xi_81 + xi_86 + xi_90; + const double forceTerm_12 = xi_84 + xi_90 + xi_91; + const double forceTerm_13 = xi_75 + xi_93 + xi_95; + const double forceTerm_14 = xi_83 + xi_92 + xi_95; + const double forceTerm_15 = xi_81 + xi_91 + xi_97; + const double forceTerm_16 = xi_84 + xi_86 + xi_97; + const double forceTerm_17 = xi_75 + xi_92 + xi_98; + const double forceTerm_18 = xi_83 + xi_93 + xi_98; + _data_pdfs_20_30_10[ctr_0] = forceTerm_0 + xi_101*0.1 + xi_104*0.0428571428571429 + xi_114*-0.5 + xi_118*0.0238095238095238 + xi_219; + _data_pdfs_20_31_10[ctr_0] = forceTerm_1 - xi_121 + xi_128 + xi_132 + xi_140 + xi_224; + _data_pdfs_20_32_10[ctr_0] = forceTerm_2 + xi_121 + xi_127 - xi_132 + xi_140 + xi_207; + _data_pdfs_20_33_10[ctr_0] = forceTerm_3 - xi_143 + xi_145 + xi_147 + xi_148 + xi_227; + _data_pdfs_20_34_10[ctr_0] = forceTerm_4 + xi_143 - xi_145 + xi_148 + xi_149 + xi_209; + _data_pdfs_20_35_10[ctr_0] = forceTerm_5 - xi_152 + xi_155 + xi_157 + xi_158 + xi_222; + _data_pdfs_20_36_10[ctr_0] = forceTerm_6 + xi_152 + xi_154 - xi_157 + xi_158 + xi_212; + _data_pdfs_20_37_10[ctr_0] = forceTerm_7 + xi_163 + xi_166 + xi_171 + xi_228; + _data_pdfs_20_38_10[ctr_0] = forceTerm_8 + xi_166 + xi_172 + xi_174 + xi_211; + _data_pdfs_20_39_10[ctr_0] = forceTerm_9 + xi_171 + xi_172 + xi_176 + xi_217; + _data_pdfs_20_310_10[ctr_0] = forceTerm_10 + xi_163 + xi_174 + xi_176 + xi_221; + _data_pdfs_20_311_10[ctr_0] = forceTerm_11 + xi_180 + xi_188 + xi_191 + xi_213; + _data_pdfs_20_312_10[ctr_0] = forceTerm_12 + xi_188 + xi_192 + xi_193 + xi_225; + _data_pdfs_20_313_10[ctr_0] = forceTerm_13 + xi_197 + xi_198 + xi_201 + xi_215; + _data_pdfs_20_314_10[ctr_0] = forceTerm_14 + xi_198 + xi_202 + xi_203 + xi_208; + _data_pdfs_20_315_10[ctr_0] = forceTerm_15 + xi_191 + xi_193 + xi_205 + xi_218; + _data_pdfs_20_316_10[ctr_0] = forceTerm_16 + xi_180 + xi_192 + xi_205 + xi_210; + _data_pdfs_20_317_10[ctr_0] = forceTerm_17 + xi_201 + xi_202 + xi_206 + xi_226; + _data_pdfs_20_318_10[ctr_0] = forceTerm_18 + xi_197 + xi_203 + xi_206 + xi_214; + } + } } - } - } + } +} } -} // namespace internal_kernel_collide namespace internal_kernel_stream { -static FUNC_PREFIX void -kernel_stream(double *RESTRICT const _data_pdfs, - double *RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, - int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, - int64_t const _stride_pdfs_tmp_2, - int64_t const _stride_pdfs_tmp_3) { - for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2; - double *RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2 * ctr_2; - double *RESTRICT _data_pdfs_tmp_20_31 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_32 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 2 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_33 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 3 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_34 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 4 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_35 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 5 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_36 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 6 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_37 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 7 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_38 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 8 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_39 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 9 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_310 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 10 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_311 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 11 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_312 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 12 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_313 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 13 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_314 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 14 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_315 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 15 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_316 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 16 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_317 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 17 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_tmp_20_318 = - _data_pdfs_tmp + _stride_pdfs_tmp_2 * ctr_2 + 18 * _stride_pdfs_tmp_3; - double *RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_tmp_20_30_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_30; - double *RESTRICT _data_pdfs_20_30_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_30; - double *RESTRICT _data_pdfs_tmp_20_31_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_31; - double *RESTRICT _data_pdfs_20_31_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_tmp_20_32_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_32; - double *RESTRICT _data_pdfs_20_32_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_tmp_20_33_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_33; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_tmp_20_34_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_34; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_tmp_20_35_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_35; - double *RESTRICT _data_pdfs_2m1_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_35; - double *RESTRICT _data_pdfs_tmp_20_36_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_36; - double *RESTRICT _data_pdfs_21_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_36; - double *RESTRICT _data_pdfs_tmp_20_37_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_37; - double *RESTRICT _data_pdfs_20_37_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_tmp_20_38_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_38; - double *RESTRICT _data_pdfs_20_38_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; - double *RESTRICT _data_pdfs_tmp_20_39_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_39; - double *RESTRICT _data_pdfs_20_39_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_tmp_20_310_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_310; - double *RESTRICT _data_pdfs_20_310_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_tmp_20_311_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_311; - double *RESTRICT _data_pdfs_2m1_311_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; - double *RESTRICT _data_pdfs_tmp_20_312_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_312; - double *RESTRICT _data_pdfs_2m1_312_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; - double *RESTRICT _data_pdfs_tmp_20_313_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_313; - double *RESTRICT _data_pdfs_2m1_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_313; - double *RESTRICT _data_pdfs_tmp_20_314_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_314; - double *RESTRICT _data_pdfs_2m1_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_314; - double *RESTRICT _data_pdfs_tmp_20_315_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_315; - double *RESTRICT _data_pdfs_21_315_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; - double *RESTRICT _data_pdfs_tmp_20_316_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_316; - double *RESTRICT _data_pdfs_21_316_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; - double *RESTRICT _data_pdfs_tmp_20_317_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_317; - double *RESTRICT _data_pdfs_21_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_317; - double *RESTRICT _data_pdfs_tmp_20_318_10 = - _stride_pdfs_tmp_1 * ctr_1 + _data_pdfs_tmp_20_318; - double *RESTRICT _data_pdfs_21_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_318; +static FUNC_PREFIX void kernel_stream(double * RESTRICT const _data_pdfs, double * RESTRICT _data_pdfs_tmp, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3, int64_t const _stride_pdfs_tmp_1, int64_t const _stride_pdfs_tmp_2, int64_t const _stride_pdfs_tmp_3) +{ + for (int64_t ctr_2 = 1; ctr_2 < _size_pdfs_2 - 1; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_tmp_20_30 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2; + double * RESTRICT _data_pdfs_20_30 = _data_pdfs + _stride_pdfs_2*ctr_2; + double * RESTRICT _data_pdfs_tmp_20_31 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + _stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_32 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 2*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_33 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 3*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_34 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 4*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_35 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 5*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_35 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_36 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 6*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_36 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_37 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 7*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_38 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 8*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_39 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 9*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_310 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 10*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_311 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 11*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_311 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_312 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 12*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_312 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_313 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 13*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_313 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_314 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 14*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_2m1_314 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_315 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 15*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_315 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_316 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 16*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_316 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_317 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 17*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_317 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_tmp_20_318 = _data_pdfs_tmp + _stride_pdfs_tmp_2*ctr_2 + 18*_stride_pdfs_tmp_3; + double * RESTRICT _data_pdfs_21_318 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 1; ctr_1 < _size_pdfs_1 - 1; ctr_1 += 1) { - for (int64_t ctr_0 = 1; - ctr_0 < (int64_t)((_size_pdfs_0 - 2) / (4)) * (4) + 1; - ctr_0 += 4) { - _mm256_storeu_pd(&_data_pdfs_tmp_20_30_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_30_10[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_31_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_31_1m1[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_32_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_32_11[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_33_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_33_10[ctr_0 + 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_34_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_34_10[ctr_0 - 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_35_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_35_10[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_36_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_36_10[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_37_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_37_1m1[ctr_0 + 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_38_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_38_1m1[ctr_0 - 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_39_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_39_11[ctr_0 + 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_310_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_20_310_11[ctr_0 - 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_311_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_311_1m1[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_312_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_312_11[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_313_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_313_10[ctr_0 + 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_314_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_2m1_314_10[ctr_0 - 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_315_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_315_1m1[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_316_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_316_11[ctr_0])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_317_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_317_10[ctr_0 + 1])); - _mm256_storeu_pd(&_data_pdfs_tmp_20_318_10[ctr_0], - _mm256_loadu_pd(&_data_pdfs_21_318_10[ctr_0 - 1])); - } - for (int64_t ctr_0 = (int64_t)((_size_pdfs_0 - 2) / (4)) * (4) + 1; - ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) { - _data_pdfs_tmp_20_30_10[ctr_0] = _data_pdfs_20_30_10[ctr_0]; - _data_pdfs_tmp_20_31_10[ctr_0] = _data_pdfs_20_31_1m1[ctr_0]; - _data_pdfs_tmp_20_32_10[ctr_0] = _data_pdfs_20_32_11[ctr_0]; - _data_pdfs_tmp_20_33_10[ctr_0] = _data_pdfs_20_33_10[ctr_0 + 1]; - _data_pdfs_tmp_20_34_10[ctr_0] = _data_pdfs_20_34_10[ctr_0 - 1]; - _data_pdfs_tmp_20_35_10[ctr_0] = _data_pdfs_2m1_35_10[ctr_0]; - _data_pdfs_tmp_20_36_10[ctr_0] = _data_pdfs_21_36_10[ctr_0]; - _data_pdfs_tmp_20_37_10[ctr_0] = _data_pdfs_20_37_1m1[ctr_0 + 1]; - _data_pdfs_tmp_20_38_10[ctr_0] = _data_pdfs_20_38_1m1[ctr_0 - 1]; - _data_pdfs_tmp_20_39_10[ctr_0] = _data_pdfs_20_39_11[ctr_0 + 1]; - _data_pdfs_tmp_20_310_10[ctr_0] = _data_pdfs_20_310_11[ctr_0 - 1]; - _data_pdfs_tmp_20_311_10[ctr_0] = _data_pdfs_2m1_311_1m1[ctr_0]; - _data_pdfs_tmp_20_312_10[ctr_0] = _data_pdfs_2m1_312_11[ctr_0]; - _data_pdfs_tmp_20_313_10[ctr_0] = _data_pdfs_2m1_313_10[ctr_0 + 1]; - _data_pdfs_tmp_20_314_10[ctr_0] = _data_pdfs_2m1_314_10[ctr_0 - 1]; - _data_pdfs_tmp_20_315_10[ctr_0] = _data_pdfs_21_315_1m1[ctr_0]; - _data_pdfs_tmp_20_316_10[ctr_0] = _data_pdfs_21_316_11[ctr_0]; - _data_pdfs_tmp_20_317_10[ctr_0] = _data_pdfs_21_317_10[ctr_0 + 1]; - _data_pdfs_tmp_20_318_10[ctr_0] = _data_pdfs_21_318_10[ctr_0 - 1]; - } + double * RESTRICT _data_pdfs_tmp_20_30_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_30; + double * RESTRICT _data_pdfs_20_30_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_30; + double * RESTRICT _data_pdfs_tmp_20_31_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_31; + double * RESTRICT _data_pdfs_20_31_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_tmp_20_32_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_32; + double * RESTRICT _data_pdfs_20_32_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_tmp_20_33_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_33; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_tmp_20_34_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_34; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_tmp_20_35_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_35; + double * RESTRICT _data_pdfs_2m1_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_35; + double * RESTRICT _data_pdfs_tmp_20_36_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_36; + double * RESTRICT _data_pdfs_21_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_36; + double * RESTRICT _data_pdfs_tmp_20_37_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_37; + double * RESTRICT _data_pdfs_20_37_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_tmp_20_38_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_38; + double * RESTRICT _data_pdfs_20_38_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_38; + double * RESTRICT _data_pdfs_tmp_20_39_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_39; + double * RESTRICT _data_pdfs_20_39_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_tmp_20_310_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_310; + double * RESTRICT _data_pdfs_20_310_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_tmp_20_311_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_311; + double * RESTRICT _data_pdfs_2m1_311_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_311; + double * RESTRICT _data_pdfs_tmp_20_312_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_312; + double * RESTRICT _data_pdfs_2m1_312_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_312; + double * RESTRICT _data_pdfs_tmp_20_313_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_313; + double * RESTRICT _data_pdfs_2m1_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_313; + double * RESTRICT _data_pdfs_tmp_20_314_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_314; + double * RESTRICT _data_pdfs_2m1_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_314; + double * RESTRICT _data_pdfs_tmp_20_315_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_315; + double * RESTRICT _data_pdfs_21_315_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_315; + double * RESTRICT _data_pdfs_tmp_20_316_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_316; + double * RESTRICT _data_pdfs_21_316_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_316; + double * RESTRICT _data_pdfs_tmp_20_317_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_317; + double * RESTRICT _data_pdfs_21_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_317; + double * RESTRICT _data_pdfs_tmp_20_318_10 = _stride_pdfs_tmp_1*ctr_1 + _data_pdfs_tmp_20_318; + double * RESTRICT _data_pdfs_21_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_318; + { + for (int64_t ctr_0 = 1; ctr_0 < (int64_t)((_size_pdfs_0 - 2) / (4)) * (4) + 1; ctr_0 += 4) + { + _mm256_storeu_pd(&_data_pdfs_tmp_20_30_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_30_10[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_31_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_31_1m1[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_32_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_32_11[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_33_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_33_10[ctr_0 + 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_34_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_34_10[ctr_0 - 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_35_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_35_10[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_36_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_36_10[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_37_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_37_1m1[ctr_0 + 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_38_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_38_1m1[ctr_0 - 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_39_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_39_11[ctr_0 + 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_310_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_20_310_11[ctr_0 - 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_311_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_311_1m1[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_312_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_312_11[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_313_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_313_10[ctr_0 + 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_314_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_2m1_314_10[ctr_0 - 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_315_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_315_1m1[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_316_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_316_11[ctr_0])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_317_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_317_10[ctr_0 + 1])); + _mm256_storeu_pd(&_data_pdfs_tmp_20_318_10[ctr_0],_mm256_loadu_pd(& _data_pdfs_21_318_10[ctr_0 - 1])); + } + for (int64_t ctr_0 = (int64_t)((_size_pdfs_0 - 2) / (4)) * (4) + 1; ctr_0 < _size_pdfs_0 - 1; ctr_0 += 1) + { + _data_pdfs_tmp_20_30_10[ctr_0] = _data_pdfs_20_30_10[ctr_0]; + _data_pdfs_tmp_20_31_10[ctr_0] = _data_pdfs_20_31_1m1[ctr_0]; + _data_pdfs_tmp_20_32_10[ctr_0] = _data_pdfs_20_32_11[ctr_0]; + _data_pdfs_tmp_20_33_10[ctr_0] = _data_pdfs_20_33_10[ctr_0 + 1]; + _data_pdfs_tmp_20_34_10[ctr_0] = _data_pdfs_20_34_10[ctr_0 - 1]; + _data_pdfs_tmp_20_35_10[ctr_0] = _data_pdfs_2m1_35_10[ctr_0]; + _data_pdfs_tmp_20_36_10[ctr_0] = _data_pdfs_21_36_10[ctr_0]; + _data_pdfs_tmp_20_37_10[ctr_0] = _data_pdfs_20_37_1m1[ctr_0 + 1]; + _data_pdfs_tmp_20_38_10[ctr_0] = _data_pdfs_20_38_1m1[ctr_0 - 1]; + _data_pdfs_tmp_20_39_10[ctr_0] = _data_pdfs_20_39_11[ctr_0 + 1]; + _data_pdfs_tmp_20_310_10[ctr_0] = _data_pdfs_20_310_11[ctr_0 - 1]; + _data_pdfs_tmp_20_311_10[ctr_0] = _data_pdfs_2m1_311_1m1[ctr_0]; + _data_pdfs_tmp_20_312_10[ctr_0] = _data_pdfs_2m1_312_11[ctr_0]; + _data_pdfs_tmp_20_313_10[ctr_0] = _data_pdfs_2m1_313_10[ctr_0 + 1]; + _data_pdfs_tmp_20_314_10[ctr_0] = _data_pdfs_2m1_314_10[ctr_0 - 1]; + _data_pdfs_tmp_20_315_10[ctr_0] = _data_pdfs_21_315_1m1[ctr_0]; + _data_pdfs_tmp_20_316_10[ctr_0] = _data_pdfs_21_316_11[ctr_0]; + _data_pdfs_tmp_20_317_10[ctr_0] = _data_pdfs_21_317_10[ctr_0 + 1]; + _data_pdfs_tmp_20_318_10[ctr_0] = _data_pdfs_21_318_10[ctr_0 - 1]; + } + } } - } - } + } } -} // namespace internal_kernel_stream +} + -const real_t MRTLatticeModelAvx::w[19] = { - 0.333333333333333, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0555555555555556, 0.0555555555555556, - 0.0555555555555556, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778, 0.0277777777777778, 0.0277777777777778, - 0.0277777777777778}; -const real_t MRTLatticeModelAvx::wInv[19] = { - 3.00000000000000, 18.0000000000000, 18.0000000000000, 18.0000000000000, - 18.0000000000000, 18.0000000000000, 18.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000, 36.0000000000000, - 36.0000000000000, 36.0000000000000, 36.0000000000000}; +const real_t MRTLatticeModelAvx::w[19] = { 0.333333333333333,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0555555555555556,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778,0.0277777777777778 }; +const real_t MRTLatticeModelAvx::wInv[19] = { 3.00000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,18.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000,36.0000000000000 }; -void MRTLatticeModelAvx::Sweep::streamCollide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); +void MRTLatticeModelAvx::Sweep::streamCollide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_streamCollide::kernel_streamCollide( - _data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, - _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, - omega_shear); - pdfs->swapDataPointers(pdfs_tmp); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & omega_even = lm.omega_even_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & omega_odd = lm.omega_odd_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_streamCollide::kernel_streamCollide(_data_force, _data_pdfs, _data_pdfs_tmp, _size_force_0, _size_force_1, _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3, omega_bulk, omega_even, omega_odd, omega_shear); + pdfs->swapDataPointers(pdfs_tmp); + } -void MRTLatticeModelAvx::Sweep::collide( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); +void MRTLatticeModelAvx::Sweep::collide( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); - auto &lm = - dynamic_cast *>(pdfs)->latticeModel(); - WALBERLA_ASSERT_EQUAL(*(lm.blockId_), block->getId()); - auto &omega_bulk = lm.omega_bulk_; - auto &omega_even = lm.omega_even_; - auto &omega_shear = lm.omega_shear_; - auto &force = lm.force_; - auto &omega_odd = lm.omega_odd_; - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(force->nrOfGhostLayers())); - double *RESTRICT const _data_force = - force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), - -cell_idx_c(numberOfGhostLayersToInclude), 0); - WALBERLA_ASSERT_GREATER_EQUAL( - force->xSizeWithGhostLayer(), - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_0 = - int64_t(cell_idx_c(force->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->ySizeWithGhostLayer(), - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_1 = - int64_t(cell_idx_c(force->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - WALBERLA_ASSERT_GREATER_EQUAL( - force->zSizeWithGhostLayer(), - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude))); - const int64_t _size_force_2 = - int64_t(cell_idx_c(force->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude)); - const int64_t _stride_force_1 = int64_t(force->yStride()); - const int64_t _stride_force_2 = int64_t(force->zStride()); - const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_kernel_collide::kernel_collide( - _data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, - _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, - _stride_pdfs_2, _stride_pdfs_3, omega_bulk, omega_even, omega_odd, - omega_shear); + auto & lm = dynamic_cast< lbm::PdfField * > (pdfs)->latticeModel(); + WALBERLA_ASSERT_EQUAL( *(lm.blockId_), block->getId() ); + + auto & omega_even = lm.omega_even_; + auto & omega_shear = lm.omega_shear_; + auto & force = lm.force_; + auto & omega_odd = lm.omega_odd_; + auto & omega_bulk = lm.omega_bulk_; + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(force->nrOfGhostLayers())); + double * RESTRICT const _data_force = force->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), -cell_idx_c(numberOfGhostLayersToInclude), 0); + WALBERLA_ASSERT_GREATER_EQUAL(force->xSizeWithGhostLayer(), int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_0 = int64_t(cell_idx_c(force->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->ySizeWithGhostLayer(), int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_1 = int64_t(cell_idx_c(force->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + WALBERLA_ASSERT_GREATER_EQUAL(force->zSizeWithGhostLayer(), int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude))); + const int64_t _size_force_2 = int64_t(cell_idx_c(force->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude)); + const int64_t _stride_force_1 = int64_t(force->yStride()); + const int64_t _stride_force_2 = int64_t(force->zStride()); + const int64_t _stride_force_3 = int64_t(1 * int64_t(force->fStride())); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_kernel_collide::kernel_collide(_data_force, _data_pdfs, _size_force_0, _size_force_1, _size_force_2, _stride_force_1, _stride_force_2, _stride_force_3, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, omega_bulk, omega_even, omega_odd, omega_shear); } -void MRTLatticeModelAvx::Sweep::stream( - IBlock *block, const uint_t numberOfGhostLayersToInclude) { - auto pdfs = block->getData>(pdfsID); - field::GhostLayerField *pdfs_tmp; - { - // Getting temporary field pdfs_tmp - auto it = cache_pdfs_.find(pdfs); - if (it != cache_pdfs_.end()) { - pdfs_tmp = *it; - } else { - pdfs_tmp = pdfs->cloneUninitialized(); - cache_pdfs_.insert(pdfs_tmp); + +void MRTLatticeModelAvx::Sweep::stream( IBlock * block, const uint_t numberOfGhostLayersToInclude ) +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + field::GhostLayerField * pdfs_tmp; + { + // Getting temporary field pdfs_tmp + auto it = cache_pdfs_.find( pdfs ); + if( it != cache_pdfs_.end() ) + { + pdfs_tmp = *it; + } + else + { + pdfs_tmp = pdfs->cloneUninitialized(); + cache_pdfs_.insert(pdfs_tmp); + } } - } - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -int_c(pdfs_tmp->nrOfGhostLayers())); - double *RESTRICT _data_pdfs_tmp = - pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, - -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_0 = - int64_t(cell_idx_c(pdfs->xSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_1 = - int64_t(cell_idx_c(pdfs->ySize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - WALBERLA_ASSERT_GREATER_EQUAL( - pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2)); - const int64_t _size_pdfs_2 = - int64_t(cell_idx_c(pdfs->zSize()) + - 2 * cell_idx_c(numberOfGhostLayersToInclude) + 2); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); - const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); - const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); - internal_kernel_stream::kernel_stream( - _data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, - _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); - pdfs->swapDataPointers(pdfs_tmp); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -int_c(pdfs_tmp->nrOfGhostLayers())); + double * RESTRICT _data_pdfs_tmp = pdfs_tmp->dataAt(-cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, -cell_idx_c(numberOfGhostLayersToInclude) - 1, 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(pdfs->xSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(pdfs->ySize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(pdfs->zSize()) + 2*cell_idx_c(numberOfGhostLayersToInclude) + 2); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + const int64_t _stride_pdfs_tmp_1 = int64_t(pdfs_tmp->yStride()); + const int64_t _stride_pdfs_tmp_2 = int64_t(pdfs_tmp->zStride()); + const int64_t _stride_pdfs_tmp_3 = int64_t(1 * int64_t(pdfs_tmp->fStride())); + internal_kernel_stream::kernel_stream(_data_pdfs, _data_pdfs_tmp, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3, _stride_pdfs_tmp_1, _stride_pdfs_tmp_2, _stride_pdfs_tmp_3); + + pdfs->swapDataPointers(pdfs_tmp); + } + } // namespace lbm } // namespace walberla + + + // Buffer Packing namespace walberla { namespace mpi { -mpi::SendBuffer &operator<<(mpi::SendBuffer &buf, - const ::walberla::lbm::MRTLatticeModelAvx &lm) { - buf << lm.currentLevel; - return buf; +mpi::SendBuffer & operator<< (mpi::SendBuffer & buf, const ::walberla::lbm::MRTLatticeModelAvx & lm) +{ + buf << lm.currentLevel; + return buf; } -mpi::RecvBuffer &operator>>(mpi::RecvBuffer &buf, - ::walberla::lbm::MRTLatticeModelAvx &lm) { - buf >> lm.currentLevel; - return buf; +mpi::RecvBuffer & operator>> (mpi::RecvBuffer & buf, ::walberla::lbm::MRTLatticeModelAvx & lm) +{ + buf >> lm.currentLevel; + return buf; } + } // namespace mpi } // namespace walberla -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic pop +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic pop #endif \ No newline at end of file diff --git a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h index f7e428c7c4d..0d287bdaa6b 100644 --- a/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h +++ b/src/walberla_bridge/generated_kernels/MRTLatticeModelAvx.h @@ -687,16 +687,101 @@ template<> struct PressureTensor { template< typename FieldPtrOrIterator > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModelAvx & /* latticeModel */, const FieldPtrOrIterator & /* it */ ) + static void get( Matrix3< real_t > & pressureTensor, const MRTLatticeModelAvx & lm, const FieldPtrOrIterator & it ) { - WALBERLA_ABORT("Not implemented"); + const auto x = it.x(); + const auto y = it.y(); + const auto z = it.z(); + + const real_t f_0 = it[0]; + const real_t f_1 = it[1]; + const real_t f_2 = it[2]; + const real_t f_3 = it[3]; + const real_t f_4 = it[4]; + const real_t f_5 = it[5]; + const real_t f_6 = it[6]; + const real_t f_7 = it[7]; + const real_t f_8 = it[8]; + const real_t f_9 = it[9]; + const real_t f_10 = it[10]; + const real_t f_11 = it[11]; + const real_t f_12 = it[12]; + const real_t f_13 = it[13]; + const real_t f_14 = it[14]; + const real_t f_15 = it[15]; + const real_t f_16 = it[16]; + const real_t f_17 = it[17]; + const real_t f_18 = it[18]; + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } template< typename PdfField_T > - static void get( Matrix3< real_t > & /* pressureTensor */, const MRTLatticeModelAvx & /* latticeModel */, const PdfField_T & /* pdf */, - const cell_idx_t /* x */, const cell_idx_t /* y */, const cell_idx_t /* z */ ) + static void get( Matrix3< real_t > & pressureTensor , const MRTLatticeModelAvx & lm , const PdfField_T & pdf, + const cell_idx_t x, const cell_idx_t y, const cell_idx_t z ) { - WALBERLA_ABORT("Not implemented"); + const real_t & xyz0 = pdf(x,y,z,0); + const real_t f_0 = pdf.getF( &xyz0, 0); + const real_t f_1 = pdf.getF( &xyz0, 1); + const real_t f_2 = pdf.getF( &xyz0, 2); + const real_t f_3 = pdf.getF( &xyz0, 3); + const real_t f_4 = pdf.getF( &xyz0, 4); + const real_t f_5 = pdf.getF( &xyz0, 5); + const real_t f_6 = pdf.getF( &xyz0, 6); + const real_t f_7 = pdf.getF( &xyz0, 7); + const real_t f_8 = pdf.getF( &xyz0, 8); + const real_t f_9 = pdf.getF( &xyz0, 9); + const real_t f_10 = pdf.getF( &xyz0, 10); + const real_t f_11 = pdf.getF( &xyz0, 11); + const real_t f_12 = pdf.getF( &xyz0, 12); + const real_t f_13 = pdf.getF( &xyz0, 13); + const real_t f_14 = pdf.getF( &xyz0, 14); + const real_t f_15 = pdf.getF( &xyz0, 15); + const real_t f_16 = pdf.getF( &xyz0, 16); + const real_t f_17 = pdf.getF( &xyz0, 17); + const real_t f_18 = pdf.getF( &xyz0, 18); + const double p_0 = f_10 + f_13 + f_14 + f_17 + f_18 + f_3 + f_4 + f_7 + f_8 + f_9; + const double p_1 = -f_10 - f_7 + f_8 + f_9; + const double p_2 = -f_13 + f_14 + f_17 - f_18; + const double p_3 = -f_10 - f_7 + f_8 + f_9; + const double p_4 = f_1 + f_10 + f_11 + f_12 + f_15 + f_16 + f_2 + f_7 + f_8 + f_9; + const double p_5 = f_11 - f_12 - f_15 + f_16; + const double p_6 = -f_13 + f_14 + f_17 - f_18; + const double p_7 = f_11 - f_12 - f_15 + f_16; + const double p_8 = f_11 + f_12 + f_13 + f_14 + f_15 + f_16 + f_17 + f_18 + f_5 + f_6; + pressureTensor[0] = p_0; + pressureTensor[1] = p_1; + pressureTensor[2] = p_2; + + pressureTensor[3] = p_3; + pressureTensor[4] = p_4; + pressureTensor[5] = p_5; + + pressureTensor[6] = p_6; + pressureTensor[7] = p_7; + pressureTensor[8] = p_8; + + } }; diff --git a/src/walberla_bridge/generated_kernels/PushPackInfo.cpp b/src/walberla_bridge/generated_kernels/PushPackInfo.cpp index ce20e466e48..429b5cb4fe0 100644 --- a/src/walberla_bridge/generated_kernels/PushPackInfo.cpp +++ b/src/walberla_bridge/generated_kernels/PushPackInfo.cpp @@ -1,2240 +1,1705 @@ -#include "PushPackInfo.h" -#include "core/DataTypes.h" -#include "core/cell/CellInterval.h" #include "stencil/Directions.h" +#include "core/cell/CellInterval.h" +#include "core/DataTypes.h" +#include "PushPackInfo.h" -#if (defined WALBERLA_CXX_COMPILER_IS_GNU) || \ - (defined WALBERLA_CXX_COMPILER_IS_CLANG) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wfloat-equal" -#pragma GCC diagnostic ignored "-Wshadow" -#pragma GCC diagnostic ignored "-Wconversion" -#pragma GCC diagnostic ignored "-Wunused-variable" +#if ( defined WALBERLA_CXX_COMPILER_IS_GNU ) || ( defined WALBERLA_CXX_COMPILER_IS_CLANG ) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wfloat-equal" +# pragma GCC diagnostic ignored "-Wshadow" +# pragma GCC diagnostic ignored "-Wconversion" +# pragma GCC diagnostic ignored "-Wunused-variable" #endif + + namespace walberla { namespace lbm { using walberla::cell::CellInterval; using walberla::stencil::Direction; + + namespace internal_pack_SW { -static FUNC_PREFIX void -pack_SW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_SW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_SW namespace internal_pack_BW { -static FUNC_PREFIX void -pack_BW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_BW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_BW namespace internal_pack_W { -static FUNC_PREFIX void -pack_W(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_W(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_W namespace internal_pack_TW { -static FUNC_PREFIX void -pack_TW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_TW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_TW namespace internal_pack_NW { -static FUNC_PREFIX void -pack_NW(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; +static FUNC_PREFIX void pack_NW(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_NW namespace internal_pack_BS { -static FUNC_PREFIX void -pack_BS(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_BS(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_BS namespace internal_pack_S { -static FUNC_PREFIX void -pack_S(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_20_32_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_32_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_S(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_20_32_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_32_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_S namespace internal_pack_TS { -static FUNC_PREFIX void -pack_TS(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_TS(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_TS namespace internal_pack_B { -static FUNC_PREFIX void -pack_B(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_2m1_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_36; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_2m1_36_10[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_B(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_2m1_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_36; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_2m1_36_10[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_B namespace internal_pack_T { -static FUNC_PREFIX void -pack_T(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_21_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_35; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_21_35_10[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_T(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_21_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_35; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_21_35_10[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_T namespace internal_pack_BN { -static FUNC_PREFIX void -pack_BN(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_BN(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_BN namespace internal_pack_N { -static FUNC_PREFIX void -pack_N(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_20_31_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_31_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_N(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_20_31_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_31_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_N namespace internal_pack_TN { -static FUNC_PREFIX void -pack_TN(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0]; +static FUNC_PREFIX void pack_TN(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0]; + } } - } - } + } +} } -} // namespace internal_pack_TN namespace internal_pack_SE { -static FUNC_PREFIX void -pack_SE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_SE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_SE namespace internal_pack_BE { -static FUNC_PREFIX void -pack_BE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_BE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_BE namespace internal_pack_E { -static FUNC_PREFIX void -pack_E(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0] = - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1] = - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2] = - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3] = - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4] = - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_E(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0] = _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1] = _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2] = _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3] = _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4] = _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_E namespace internal_pack_TE { -static FUNC_PREFIX void -pack_TE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_TE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_TE namespace internal_pack_NE { -static FUNC_PREFIX void -pack_NE(double *RESTRICT _data_buffer, double *RESTRICT const _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0] = - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0]; +static FUNC_PREFIX void pack_NE(double * RESTRICT _data_buffer, double * RESTRICT const _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0] = _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0]; + } } - } - } + } +} } -} // namespace internal_pack_NE + + namespace internal_unpack_SW { -static FUNC_PREFIX void -unpack_SW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_SW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_SW namespace internal_unpack_BW { -static FUNC_PREFIX void -unpack_BW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BW namespace internal_unpack_W { -static FUNC_PREFIX void -unpack_W(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_33 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 3 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_20_33_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_33; - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_33_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_W(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_33 = _data_pdfs + _stride_pdfs_2*ctr_2 + 3*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_20_33_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_33; + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_33_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_W namespace internal_unpack_TW { -static FUNC_PREFIX void -unpack_TW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TW namespace internal_unpack_NW { -static FUNC_PREFIX void -unpack_NW(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_NW(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_NW namespace internal_unpack_BS { -static FUNC_PREFIX void -unpack_BS(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BS(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BS namespace internal_unpack_S { -static FUNC_PREFIX void -unpack_S(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_39 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 9 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_32 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 2 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_39_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_20_32_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_39_1m1[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_32_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_S(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_39 = _data_pdfs + _stride_pdfs_2*ctr_2 + 9*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_32 = _data_pdfs + _stride_pdfs_2*ctr_2 + 2*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_39_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_39; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_20_32_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_32; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_39_1m1[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_32_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_S namespace internal_unpack_TS { -static FUNC_PREFIX void -unpack_TS(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TS(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TS namespace internal_unpack_B { -static FUNC_PREFIX void -unpack_B(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 17 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 16 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 6 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_317_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_317; - double *RESTRICT _data_pdfs_2m1_316_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; - double *RESTRICT _data_pdfs_2m1_36_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_36; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_317_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_316_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_2m1_36_10[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_B(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_317 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 17*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_316 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 16*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_36 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 6*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_317_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_317; + double * RESTRICT _data_pdfs_2m1_316_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_2m1_316; + double * RESTRICT _data_pdfs_2m1_36_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_36; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_317_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_316_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_2m1_36_10[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_B namespace internal_unpack_T { -static FUNC_PREFIX void -unpack_T(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 13 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 12 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 5 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_313_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_313; - double *RESTRICT _data_pdfs_21_312_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; - double *RESTRICT _data_pdfs_21_35_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_35; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_313_10[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_21_312_1m1[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_21_35_10[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_T(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_313 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 13*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_312 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 12*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_35 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 5*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_313_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_313; + double * RESTRICT _data_pdfs_21_312_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_21_312; + double * RESTRICT _data_pdfs_21_35_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_35; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_313_10[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_21_312_1m1[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_21_35_10[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_T namespace internal_unpack_BN { -static FUNC_PREFIX void -unpack_BN(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BN(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BN namespace internal_unpack_N { -static FUNC_PREFIX void -unpack_N(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_37 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 7 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 15 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_31 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_37_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; - double *RESTRICT _data_pdfs_2m1_315_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; - double *RESTRICT _data_pdfs_20_31_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_37_11[_stride_pdfs_0 * ctr_0 - _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_315_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_31_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_N(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_37 = _data_pdfs + _stride_pdfs_2*ctr_2 + 7*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_315 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 15*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_31 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_3; + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_37_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_37; + double * RESTRICT _data_pdfs_2m1_315_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_2m1_315; + double * RESTRICT _data_pdfs_20_31_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_31; + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_37_11[_stride_pdfs_0*ctr_0 - _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_315_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_31_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_N namespace internal_unpack_TN { -static FUNC_PREFIX void -unpack_TN(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 11 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_311_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_311_11[_stride_pdfs_0 * ctr_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TN(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_311 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 11*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_311_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_21_311; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_311_11[_stride_pdfs_0*ctr_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TN namespace internal_unpack_SE { -static FUNC_PREFIX void -unpack_SE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_SE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_SE namespace internal_unpack_BE { -static FUNC_PREFIX void -unpack_BE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_BE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_BE namespace internal_unpack_E { -static FUNC_PREFIX void -unpack_E(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_310 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 10 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2 * ctr_2 - - _stride_pdfs_2 + 18 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_34 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 4 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_310_1m1 = - _stride_pdfs_1 * ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; - double *RESTRICT _data_pdfs_2m1_318_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_2m1_318; - double *RESTRICT _data_pdfs_20_34_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_20_34; - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_310_1m1[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0]; - _data_pdfs_2m1_318_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 1]; - _data_pdfs_20_34_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 2]; - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 3]; - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[5 * _size_pdfs_0 * _size_pdfs_1 * ctr_2 + - 5 * _size_pdfs_0 * ctr_1 + 5 * ctr_0 + 4]; +static FUNC_PREFIX void unpack_E(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_310 = _data_pdfs + _stride_pdfs_2*ctr_2 + 10*_stride_pdfs_3; + double * RESTRICT _data_pdfs_2m1_318 = _data_pdfs + _stride_pdfs_2*ctr_2 - _stride_pdfs_2 + 18*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_34 = _data_pdfs + _stride_pdfs_2*ctr_2 + 4*_stride_pdfs_3; + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_310_1m1 = _stride_pdfs_1*ctr_1 - _stride_pdfs_1 + _data_pdfs_20_310; + double * RESTRICT _data_pdfs_2m1_318_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_2m1_318; + double * RESTRICT _data_pdfs_20_34_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_20_34; + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_310_1m1[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0]; + _data_pdfs_2m1_318_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 1]; + _data_pdfs_20_34_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 2]; + _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 3]; + _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[5*_size_pdfs_0*_size_pdfs_1*ctr_2 + 5*_size_pdfs_0*ctr_1 + 5*ctr_0 + 4]; + } } - } - } + } +} } -} // namespace internal_unpack_E namespace internal_unpack_TE { -static FUNC_PREFIX void -unpack_TE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2 * ctr_2 + - _stride_pdfs_2 + 14 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_21_314_10 = - _stride_pdfs_1 * ctr_1 + _data_pdfs_21_314; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_21_314_10[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_TE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_21_314 = _data_pdfs + _stride_pdfs_2*ctr_2 + _stride_pdfs_2 + 14*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_21_314_10 = _stride_pdfs_1*ctr_1 + _data_pdfs_21_314; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_21_314_10[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } - } - } + } +} } -} // namespace internal_unpack_TE namespace internal_unpack_NE { -static FUNC_PREFIX void -unpack_NE(double *RESTRICT const _data_buffer, double *RESTRICT _data_pdfs, - int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, - int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, - int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, - int64_t const _stride_pdfs_3) { - for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) { - double *RESTRICT _data_pdfs_20_38 = - _data_pdfs + _stride_pdfs_2 * ctr_2 + 8 * _stride_pdfs_3; - for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) { - double *RESTRICT _data_pdfs_20_38_11 = - _stride_pdfs_1 * ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; - for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) { - _data_pdfs_20_38_11[_stride_pdfs_0 * ctr_0 + _stride_pdfs_0] = - _data_buffer[_size_pdfs_0 * _size_pdfs_1 * ctr_2 + - _size_pdfs_0 * ctr_1 + ctr_0]; +static FUNC_PREFIX void unpack_NE(double * RESTRICT const _data_buffer, double * RESTRICT _data_pdfs, int64_t const _size_pdfs_0, int64_t const _size_pdfs_1, int64_t const _size_pdfs_2, int64_t const _stride_pdfs_0, int64_t const _stride_pdfs_1, int64_t const _stride_pdfs_2, int64_t const _stride_pdfs_3) +{ + for (int64_t ctr_2 = 0; ctr_2 < _size_pdfs_2; ctr_2 += 1) + { + double * RESTRICT _data_pdfs_20_38 = _data_pdfs + _stride_pdfs_2*ctr_2 + 8*_stride_pdfs_3; + for (int64_t ctr_1 = 0; ctr_1 < _size_pdfs_1; ctr_1 += 1) + { + double * RESTRICT _data_pdfs_20_38_11 = _stride_pdfs_1*ctr_1 + _stride_pdfs_1 + _data_pdfs_20_38; + for (int64_t ctr_0 = 0; ctr_0 < _size_pdfs_0; ctr_0 += 1) + { + _data_pdfs_20_38_11[_stride_pdfs_0*ctr_0 + _stride_pdfs_0] = _data_buffer[_size_pdfs_0*_size_pdfs_1*ctr_2 + _size_pdfs_0*ctr_1 + ctr_0]; + } } + } +} +} + + + +void PushPackInfo::pack(Direction dir, unsigned char * byte_buffer, IBlock * block) const +{ + double * buffer = reinterpret_cast(byte_buffer); + + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + CellInterval ci; + pdfs->getSliceBeforeGhostLayer(dir, ci, 1, false); + + switch( dir ) + { + case stencil::SW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_SW::pack_SW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BW::pack_BW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::W: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_W::pack_W(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TW::pack_TW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NW: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_NW::pack_NW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BS: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BS::pack_BS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::S: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_S::pack_S(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TS: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TS::pack_TS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::B: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_B::pack_B(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::T: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_T::pack_T(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BN: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BN::pack_BN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::N: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_N::pack_N(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TN: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TN::pack_TN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::SE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_SE::pack_SE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_BE::pack_BE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::E: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_E::pack_E(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_TE::pack_TE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NE: + { + double * RESTRICT _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT const _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_pack_NE::pack_NE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + + default: + WALBERLA_ASSERT(false); + } +} + + +void PushPackInfo::unpack(Direction dir, unsigned char * byte_buffer, IBlock * block) const +{ + double * buffer = reinterpret_cast(byte_buffer); + + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + CellInterval ci; + pdfs->getGhostRegion(dir, ci, 1, false); + auto communciationDirection = stencil::inverseDir[dir]; + + switch( communciationDirection ) + { + case stencil::SW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_SW::unpack_SW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BW::unpack_BW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::W: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_W::unpack_W(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TW::unpack_TW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NW: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_NW::unpack_NW(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BS: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BS::unpack_BS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::S: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_S::unpack_S(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TS: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TS::unpack_TS(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::B: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_B::unpack_B(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::T: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_T::unpack_T(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BN: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BN::unpack_BN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::N: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_N::unpack_N(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TN: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TN::unpack_TN(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::SE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_SE::unpack_SE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::BE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_BE::unpack_BE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::E: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_E::unpack_E(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::TE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_TE::unpack_TE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + case stencil::NE: + { + double * RESTRICT const _data_buffer = buffer; + WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); + WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); + double * RESTRICT _data_pdfs = pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), int64_t(cell_idx_c(ci.xSize()) + 0)); + const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), int64_t(cell_idx_c(ci.ySize()) + 0)); + const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); + WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), int64_t(cell_idx_c(ci.zSize()) + 0)); + const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); + const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); + const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); + const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); + const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); + internal_unpack_NE::unpack_NE(_data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); + break; + } + + + default: + WALBERLA_ASSERT(false); } - } -} -} // namespace internal_unpack_NE - -void PushPackInfo::pack(Direction dir, unsigned char *byte_buffer, - IBlock *block) const { - double *buffer = reinterpret_cast(byte_buffer); - - auto pdfs = block->getData>(pdfsID); - - CellInterval ci; - pdfs->getSliceBeforeGhostLayer(dir, ci, 1, false); - - switch (dir) { - case stencil::SW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_SW::pack_SW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BW::pack_BW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::W: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_W::pack_W(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TW::pack_TW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NW: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_NW::pack_NW(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BS: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BS::pack_BS(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::S: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_S::pack_S(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TS: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TS::pack_TS(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::B: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_B::pack_B(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::T: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_T::pack_T(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BN: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BN::pack_BN(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::N: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_N::pack_N(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TN: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TN::pack_TN(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::SE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_SE::pack_SE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_BE::pack_BE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::E: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_E::pack_E(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_TE::pack_TE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NE: { - double *RESTRICT _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT const _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_pack_NE::pack_NE(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - default: - WALBERLA_ASSERT(false); - } -} - -void PushPackInfo::unpack(Direction dir, unsigned char *byte_buffer, - IBlock *block) const { - double *buffer = reinterpret_cast(byte_buffer); - - auto pdfs = block->getData>(pdfsID); - - CellInterval ci; - pdfs->getGhostRegion(dir, ci, 1, false); - auto communciationDirection = stencil::inverseDir[dir]; - - switch (communciationDirection) { - case stencil::SW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_SW::unpack_SW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BW::unpack_BW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::W: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_W::unpack_W(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TW::unpack_TW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NW: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_NW::unpack_NW( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BS: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BS::unpack_BS( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::S: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_S::unpack_S(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TS: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TS::unpack_TS( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::B: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_B::unpack_B(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::T: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_T::unpack_T(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BN: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BN::unpack_BN( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::N: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_N::unpack_N(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TN: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TN::unpack_TN( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::SE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_SE::unpack_SE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::BE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_BE::unpack_BE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::E: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_E::unpack_E(_data_buffer, _data_pdfs, _size_pdfs_0, - _size_pdfs_1, _size_pdfs_2, _stride_pdfs_0, - _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::TE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_TE::unpack_TE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - case stencil::NE: { - double *RESTRICT const _data_buffer = buffer; - WALBERLA_ASSERT_GREATER_EQUAL(ci.xMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.yMin(), -int_c(pdfs->nrOfGhostLayers())); - WALBERLA_ASSERT_GREATER_EQUAL(ci.zMin(), -int_c(pdfs->nrOfGhostLayers())); - double *RESTRICT _data_pdfs = - pdfs->dataAt(ci.xMin(), ci.yMin(), ci.zMin(), 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->xSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.xSize()) + 0)); - const int64_t _size_pdfs_0 = int64_t(cell_idx_c(ci.xSize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->ySizeWithGhostLayer(), - int64_t(cell_idx_c(ci.ySize()) + 0)); - const int64_t _size_pdfs_1 = int64_t(cell_idx_c(ci.ySize()) + 0); - WALBERLA_ASSERT_GREATER_EQUAL(pdfs->zSizeWithGhostLayer(), - int64_t(cell_idx_c(ci.zSize()) + 0)); - const int64_t _size_pdfs_2 = int64_t(cell_idx_c(ci.zSize()) + 0); - const int64_t _stride_pdfs_0 = int64_t(pdfs->xStride()); - const int64_t _stride_pdfs_1 = int64_t(pdfs->yStride()); - const int64_t _stride_pdfs_2 = int64_t(pdfs->zStride()); - const int64_t _stride_pdfs_3 = int64_t(1 * int64_t(pdfs->fStride())); - internal_unpack_NE::unpack_NE( - _data_buffer, _data_pdfs, _size_pdfs_0, _size_pdfs_1, _size_pdfs_2, - _stride_pdfs_0, _stride_pdfs_1, _stride_pdfs_2, _stride_pdfs_3); - break; - } - - default: - WALBERLA_ASSERT(false); - } -} - -uint_t PushPackInfo::size(stencil::Direction dir, const IBlock *block) const { - auto pdfs = block->getData>(pdfsID); - - CellInterval ci; - pdfs->getGhostRegion(dir, ci, 1, false); - - uint_t elementsPerCell = 0; - - switch (dir) { - case stencil::SW: - elementsPerCell = 1; - break; - - case stencil::BW: - elementsPerCell = 1; - break; - - case stencil::W: - elementsPerCell = 5; - break; - - case stencil::TW: - elementsPerCell = 1; - break; - - case stencil::NW: - elementsPerCell = 1; - break; - - case stencil::BS: - elementsPerCell = 1; - break; - - case stencil::S: - elementsPerCell = 5; - break; - - case stencil::TS: - elementsPerCell = 1; - break; - - case stencil::B: - elementsPerCell = 5; - break; - - case stencil::T: - elementsPerCell = 5; - break; - - case stencil::BN: - elementsPerCell = 1; - break; - - case stencil::N: - elementsPerCell = 5; - break; - - case stencil::TN: - elementsPerCell = 1; - break; - - case stencil::SE: - elementsPerCell = 1; - break; - - case stencil::BE: - elementsPerCell = 1; - break; - - case stencil::E: - elementsPerCell = 5; - break; - - case stencil::TE: - elementsPerCell = 1; - break; - - case stencil::NE: - elementsPerCell = 1; - break; - - default: - elementsPerCell = 0; - } - return ci.numCells() * elementsPerCell * sizeof(double); } + +uint_t PushPackInfo::size(stencil::Direction dir, const IBlock * block) const +{ + auto pdfs = block->getData< field::GhostLayerField >(pdfsID); + + CellInterval ci; + pdfs->getGhostRegion(dir, ci, 1, false); + + uint_t elementsPerCell = 0; + + switch( dir ) + { + case stencil::SW: + elementsPerCell = 1; + break; + + case stencil::BW: + elementsPerCell = 1; + break; + + case stencil::W: + elementsPerCell = 5; + break; + + case stencil::TW: + elementsPerCell = 1; + break; + + case stencil::NW: + elementsPerCell = 1; + break; + + case stencil::BS: + elementsPerCell = 1; + break; + + case stencil::S: + elementsPerCell = 5; + break; + + case stencil::TS: + elementsPerCell = 1; + break; + + case stencil::B: + elementsPerCell = 5; + break; + + case stencil::T: + elementsPerCell = 5; + break; + + case stencil::BN: + elementsPerCell = 1; + break; + + case stencil::N: + elementsPerCell = 5; + break; + + case stencil::TN: + elementsPerCell = 1; + break; + + case stencil::SE: + elementsPerCell = 1; + break; + + case stencil::BE: + elementsPerCell = 1; + break; + + case stencil::E: + elementsPerCell = 5; + break; + + case stencil::TE: + elementsPerCell = 1; + break; + + case stencil::NE: + elementsPerCell = 1; + break; + + default: + elementsPerCell = 0; + } + return ci.numCells() * elementsPerCell * sizeof( double ); +} + + + } // namespace lbm } // namespace walberla \ No newline at end of file From c06f4023868b08758ec2cc5d3b0d7c407b654ca3 Mon Sep 17 00:00:00 2001 From: itischler Date: Fri, 28 May 2021 10:26:44 +0200 Subject: [PATCH 5/7] Added correction factor for the pressure tensor --- src/core/grid_based_algorithms/lb_interface.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/core/grid_based_algorithms/lb_interface.cpp b/src/core/grid_based_algorithms/lb_interface.cpp index 096e8cf14c2..59db4b0a881 100644 --- a/src/core/grid_based_algorithms/lb_interface.cpp +++ b/src/core/grid_based_algorithms/lb_interface.cpp @@ -518,11 +518,12 @@ lb_lbnode_get_pressure_tensor(const Utils::Vector3i &ind) { ::Communication::Result::one_rank, Walberla::get_node_pressure_tensor, ind); - // reverts the correction done by walberla - // auto const revert_factor = - // lb_lbfluid_get_viscosity() / (lb_lbfluid_get_viscosity() + 1.0 - // / 6.0); stress[1] /= revert_factor; stress[3] /= revert_factor; - // stress[4] /= revert_factor; + // correction factor + auto const adjust_factor = + lb_lbfluid_get_viscosity() / (lb_lbfluid_get_viscosity() + 1.0 / 6.0); + stress[1] *= adjust_factor; + stress[3] *= adjust_factor; + stress[4] *= adjust_factor; return stress; } From 9b96617d055606470f076f8848efcd29cdcba296 Mon Sep 17 00:00:00 2001 From: itischler Date: Fri, 28 May 2021 10:27:34 +0200 Subject: [PATCH 6/7] reactivated the testing of the pressure tensor --- testsuite/python/lb_shear.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/testsuite/python/lb_shear.py b/testsuite/python/lb_shear.py index c1e73e43f8f..3ce39b27a83 100644 --- a/testsuite/python/lb_shear.py +++ b/testsuite/python/lb_shear.py @@ -17,7 +17,7 @@ import unittest as ut import unittest_decorators as utx import numpy as np -# import math WALBERLA TODO +import math import espressomd.lb import espressomd.lbboundaries @@ -144,9 +144,9 @@ def check_profile(self, shear_plane_normal, shear_direction): # speed of sound of the LB fluid in MD units (agrid/tau is due to # LB->MD unit conversion) -# speed_of_sound = 1. / math.sqrt(3.) * self.lbf.agrid / self.lbf.tau + speed_of_sound = 1. / math.sqrt(3.) * self.lbf.agrid / self.lbf.tau # equation of state for the LB fluid -# p_eq = speed_of_sound**2.0 * DENS + p_eq = speed_of_sound**2.0 * DENS # see Eq. 1.15 and 1.29 in # Krüger, Timm, et al. "The lattice Boltzmann method." Springer International Publishing 10 (2017): 978-3. # and @@ -157,16 +157,15 @@ def check_profile(self, shear_plane_normal, shear_direction): # '*' denotes the outer product and 'u' is the velocity field # NOTE: the so called stress property of the fluid is actually the # pressure tensor not the viscous stress tensor! -# shear_rate = SHEAR_VELOCITY / H + shear_rate = SHEAR_VELOCITY / H dynamic_viscosity = self.lbf.viscosity * DENS -# p_expected = p_eq * np.identity(3) - dynamic_viscosity * shear_rate * ( -# np.outer(shear_plane_normal, shear_direction) + np.transpose(np.outer(shear_plane_normal, shear_direction))) -# for n in (2, 3, 4), (3, 4, 2), (5, 4, 3): -# node_pressure_tensor = np.copy( -# self.lbf[n[0], n[1], n[2]].pressure_tensor) -# WALBERLA TODO -# np.testing.assert_allclose(node_pressure_tensor, -# p_expected, atol=1E-5, rtol=5E-3) + p_expected = p_eq * np.identity(3) - dynamic_viscosity * shear_rate * ( + np.outer(shear_plane_normal, shear_direction) + np.transpose(np.outer(shear_plane_normal, shear_direction))) + for n in (2, 3, 4), (3, 4, 2), (5, 4, 3): + node_pressure_tensor = np.copy( + self.lbf[n[0], n[1], n[2]].pressure_tensor) + np.testing.assert_allclose(node_pressure_tensor, + p_expected, atol=1E-5, rtol=5E-3) np.testing.assert_allclose( np.copy(wall1.get_force()), From ef74f2bd874c4e3258d614fcb7173ea88ce34b8b Mon Sep 17 00:00:00 2001 From: itischler Date: Fri, 28 May 2021 11:53:38 +0200 Subject: [PATCH 7/7] formatting --- src/core/grid_based_algorithms/lb_interface.hpp | 3 ++- .../grid_based_algorithms/lb_walberla_instance.cpp | 3 ++- .../grid_based_algorithms/lb_walberla_instance.hpp | 3 ++- src/python/espressomd/lb.pxd | 3 ++- src/python/espressomd/lb.pyx | 14 +++++++++++++- src/walberla_bridge/LBWalberlaBase.hpp | 3 ++- .../LBWalberlaD3Q19FluctuatingMRT.hpp | 3 ++- src/walberla_bridge/LBWalberlaD3Q19MRT.hpp | 3 ++- src/walberla_bridge/lb_walberla_init.cpp | 1 - src/walberla_bridge/lb_walberla_init.hpp | 1 - testsuite/python/lb_walberla.py | 1 - 11 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/core/grid_based_algorithms/lb_interface.hpp b/src/core/grid_based_algorithms/lb_interface.hpp index e5fc9cdf231..119386d6466 100644 --- a/src/core/grid_based_algorithms/lb_interface.hpp +++ b/src/core/grid_based_algorithms/lb_interface.hpp @@ -246,4 +246,5 @@ void lb_lbfluid_add_force_at_pos(const Utils::Vector3d &pos, */ double lb_lbfluid_get_interpolated_density(const Utils::Vector3d &pos); -#endif \ No newline at end of file +#endif + diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.cpp b/src/core/grid_based_algorithms/lb_walberla_instance.cpp index a3693d59f2e..00fe4cff671 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.cpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.cpp @@ -109,4 +109,5 @@ void mpi_destruct_lb_walberla() { lb_lbfluid_set_lattice_switch(ActiveLB::NONE); Communication::mpiCallbacks().call_all(destruct_lb_walberla); } -#endif \ No newline at end of file +#endif + diff --git a/src/core/grid_based_algorithms/lb_walberla_instance.hpp b/src/core/grid_based_algorithms/lb_walberla_instance.hpp index 0b85c67c1ab..5cc61f51892 100644 --- a/src/core/grid_based_algorithms/lb_walberla_instance.hpp +++ b/src/core/grid_based_algorithms/lb_walberla_instance.hpp @@ -63,4 +63,5 @@ void mpi_destruct_lb_walberla(); #endif // LB_WALBERLA -#endif \ No newline at end of file +#endif + diff --git a/src/python/espressomd/lb.pxd b/src/python/espressomd/lb.pxd index d994a92e402..a28d5b4e12d 100644 --- a/src/python/espressomd/lb.pxd +++ b/src/python/espressomd/lb.pxd @@ -209,4 +209,5 @@ cdef inline python_lbfluid_set_gamma(p_gamma) except +: else: c_gamma = p_gamma # call c-function - lb_lbcoupling_set_gamma(c_gamma) \ No newline at end of file + lb_lbcoupling_set_gamma(c_gamma) + diff --git a/src/python/espressomd/lb.pyx b/src/python/espressomd/lb.pyx index aa8ce7bddb5..4a07c530e95 100644 --- a/src/python/espressomd/lb.pyx +++ b/src/python/espressomd/lb.pyx @@ -118,6 +118,7 @@ IF LB_WALBERLA: class VTKOutputAutomatic(VTKOutput): """ Automatic VTK callback. Can be disabled at any time and re-enabled later. + Please note that the internal VTK counter is no longer incremented when the callback is disabled, which means the number of LB steps between two frames will not always be an integer multiple of delta_N. @@ -172,6 +173,7 @@ def _construct(cls, params): cdef class HydrodynamicInteraction(Actor): """ Base class for LB implementations. + Parameters ---------- agrid : :obj:`float` @@ -277,14 +279,17 @@ cdef class HydrodynamicInteraction(Actor): def get_interpolated_velocity(self, pos): """Get LB fluid velocity at specified position. + Parameters ---------- pos : (3,) array_like of :obj:`float` The position at which velocity is requested. + Returns ------- v : (3,) array_like :obj:`float` The LB fluid velocity at ``pos``. + """ cdef Vector3d p @@ -295,12 +300,14 @@ cdef class HydrodynamicInteraction(Actor): def add_force_at_pos(self, pos, force): """Adds a force to the fluid at given position + Parameters ---------- pos : (3,) array_like of :obj:`float` The position at which the force will be added. force : (3,) array_like of :obj:`float` The force vector which will be distributed at the position. + """ cdef Vector3d p cdef Vector3d f @@ -431,8 +438,10 @@ IF LB_WALBERLA: base_folder='vtk_out', prefix='simulation_step'): """ Create a VTK observable. + Files are written to ``//_*.vtu``. Summary is written to ``/.pvd``. + Parameters ---------- identifier : :obj:`str` @@ -598,6 +607,7 @@ class LBSlice: def _add_lb_slice_properties(): """ Automatically add all of LBFluidRoutines's properties to LBSlice. + """ def set_attribute(lb_slice, value, attribute): @@ -631,6 +641,7 @@ def _add_lb_slice_properties(): """ Getter function that copies attribute from every member of lb_slice into an array (if possible). + """ indices = [lb_slice.x_indices, lb_slice.y_indices, lb_slice.z_indices] @@ -655,4 +666,5 @@ def _add_lb_slice_properties(): setattr(LBSlice, attribute_name, new_property) -_add_lb_slice_properties() \ No newline at end of file +_add_lb_slice_properties() + diff --git a/src/walberla_bridge/LBWalberlaBase.hpp b/src/walberla_bridge/LBWalberlaBase.hpp index 516d19a00fe..8d79463ce39 100644 --- a/src/walberla_bridge/LBWalberlaBase.hpp +++ b/src/walberla_bridge/LBWalberlaBase.hpp @@ -178,4 +178,5 @@ enum class OutputVTK : unsigned { pressure_tensor = 1u << 2u, }; -#endif // LB_WALBERLA_H \ No newline at end of file +#endif // LB_WALBERLA_H + diff --git a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp index 645e37299ad..61354c681ff 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19FluctuatingMRT.hpp @@ -74,4 +74,5 @@ class LBWalberlaD3Q19FluctuatingMRT : public LBWalberlaImpl { }; } // namespace walberla -#undef LatticeModelName \ No newline at end of file +#undef LatticeModelName + diff --git a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp index 23502324b2a..312209009b8 100644 --- a/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp +++ b/src/walberla_bridge/LBWalberlaD3Q19MRT.hpp @@ -48,4 +48,5 @@ class LBWalberlaD3Q19MRT : public LBWalberlaImpl { } // namespace walberla -#undef LatticeModelName \ No newline at end of file +#undef LatticeModelName + diff --git a/src/walberla_bridge/lb_walberla_init.cpp b/src/walberla_bridge/lb_walberla_init.cpp index bb04d4f666d..aae38ed449a 100644 --- a/src/walberla_bridge/lb_walberla_init.cpp +++ b/src/walberla_bridge/lb_walberla_init.cpp @@ -32,7 +32,6 @@ void walberla_mpi_init() { walberla::mpi::Environment(argc, argv); } - LBWalberlaBase *new_lb_walberla(double viscosity, double density, const Utils::Vector3i &grid_dimensions, const Utils::Vector3i &node_grid, double kT, diff --git a/src/walberla_bridge/lb_walberla_init.hpp b/src/walberla_bridge/lb_walberla_init.hpp index dc1ffd53357..5574e5024c6 100644 --- a/src/walberla_bridge/lb_walberla_init.hpp +++ b/src/walberla_bridge/lb_walberla_init.hpp @@ -26,7 +26,6 @@ /** @brief Initialize Walberla's MPI manager */ void walberla_mpi_init(); - LBWalberlaBase *new_lb_walberla(double viscosity, double density, const Utils::Vector3i &grid_dimensions, const Utils::Vector3i &node_grid, double kT, diff --git a/testsuite/python/lb_walberla.py b/testsuite/python/lb_walberla.py index e89ea7629eb..50de3e1fcf8 100644 --- a/testsuite/python/lb_walberla.py +++ b/testsuite/python/lb_walberla.py @@ -34,7 +34,6 @@ def test(self): s.time_step = 0.2 s.cell_system.skin = 0. dens_init = 1.3 - lbf = LBFluidWalberla( agrid=.6, dens=dens_init,